Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2616677 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.014912221 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.014292194 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1201 1202 1203 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.917728542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.934970473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936222468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936317221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936330581Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936348669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"Crucible b4123650-5acb-405b-85dc-5a00d08086ff has session id: 933f6b1b-a467-4383-b21d-0ff5158d1ed1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936857445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Upstairs opts: Upstairs UUID: b4123650-5acb-405b-85dc-5a00d08086ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936928Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Crucible stats registered with UUID: b4123650-5acb-405b-85dc-5a00d08086ff","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.936945249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:31:54.936958948Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937475532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937525709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937541518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937736065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937789051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"b4123650-5acb-405b-85dc-5a00d08086ff active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937844387Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937858056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937872185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937885974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937902993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937917552Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.937931751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53163)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.938191003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59974)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.938218202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:32795)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.93823441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940030368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940090284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940111462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940660935Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940714001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940776897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940793076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940809844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940824993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940839952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940856391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.94087139Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940885829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940902018Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940915607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940929706Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940943095Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940958264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"":"downstairs","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940972483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940985502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"0","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.940999542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"1","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941013551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"client":"2","":"downstairs","session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.9410277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941041449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"b4123650-5acb-405b-85dc-5a00d08086ff is now active with session: 933f6b1b-a467-4383-b21d-0ff5158d1ed1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941056978Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"Wait for a query_work_queue command to finish before sending IO 933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941086776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206,"session_id":"933f6b1b-a467-4383-b21d-0ff5158d1ed1"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941100605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} {"msg":"Activated sub_volume b4123650-5acb-405b-85dc-5a00d08086ff","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:54.941112704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:31:59.010341684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1206} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.036945146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.048676394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049009251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049046739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049059368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049077567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"Crucible 464ea056-a828-44f0-8234-937869b80790 has session id: a9792c73-127d-48e4-8191-c7bad15a338b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049433222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Upstairs opts: Upstairs UUID: 464ea056-a828-44f0-8234-937869b80790, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049460641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Crucible stats registered with UUID: 464ea056-a828-44f0-8234-937869b80790","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.04947523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:31:59.049488609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049691925Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049725292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049740911Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049845194Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049862953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"464ea056-a828-44f0-8234-937869b80790 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049879662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049892741Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.04990645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049920579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049935098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049950377Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.049964786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56468)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.050025392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58610)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.05005424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63242)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.050071879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.050746913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.050774981Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.05079189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051143656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051161174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051185743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051200922Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.05121743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051232279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051248308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051266057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051281376Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051296445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051312934Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051326393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-16T21:31:59.051340502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051359361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.0513729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"":"downstairs","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051389359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051402498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"0","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051416457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"1","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051430466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"client":"2","":"downstairs","session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051444745Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051460914Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"464ea056-a828-44f0-8234-937869b80790 is now active with session: a9792c73-127d-48e4-8191-c7bad15a338b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051476013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051488772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208,"session_id":"a9792c73-127d-48e4-8191-c7bad15a338b"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.051501901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} {"msg":"Activated sub_volume 464ea056-a828-44f0-8234-937869b80790","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:31:59.05151346Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 337 len 12288 data: 2 2 2 02/30 Write block 96 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 268 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 86 len 28672 05/30 Write block 434 len 20480 data: 2 2 2 2 2 06/30 Read block 450 len 12288 07/30 Write block 583 len 12288 data: 2 2 2 08/30 Write block 354 len 32768 data: 2 2 2 2 2 2 2 2 09/30 Read block 484 len 20480 10/30 Read block 412 len 12288 11/30 Read block 499 len 40960 12/30 Write block 482 len 8192 data: 2 2 13/30 Write block 513 len 40960 data: 2 2 2 2 2 2 2 2 2 2 14/30 Read block 472 len 12288 15/30 Write block 284 len 20480 data: 2 2 2 2 2 16/30 Read block 92 len 8192 17/30 Read block 305 len 24576 18/30 Read block 214 len 24576 19/30 Write block 35 len 24576 data: 2 2 2 2 2 2 20/30 Read block 579 len 4096 21/30 Write block 414 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Read block 171 len 4096 23/30 Write block 361 len 8192 data: 3 2 24/30 Read block 359 len 20480 25/30 Write block 198 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Read block 286 len 8192 27/30 Write block 424 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Write block 93 len 24576 data: 2 2 2 3 3 3 29/30 Read block 192 len 4096 30/30 Read block 315 len 24576 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:31:59.063948581Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 T F T <--- 2 040-059 1 1 1 1 1 1 T F T <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 T T T 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T F T <--- 10 200-219 1 1 1 1 1 1 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 T F T <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 T F T <--- 18 360-379 1 1 1 1 1 1 T F T <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T F T <--- 21 420-439 1 1 1 1 1 1 T F T <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T F T <--- 25 500-519 1 1 1 1 1 1 T F T <--- 26 520-539 1 1 1 1 1 1 T F T <--- 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 T F T <--- Max gen: 1, Max flush: 1 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 2 2 2 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 2, Max flush: 2 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.549379031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561045864Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561382891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561424468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561437617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561455736Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"Crucible b7ea5033-7428-4d24-baa0-ce80fa7cdad9 has session id: 349371ec-e593-4007-a87c-e036ded4584b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561769635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Upstairs opts: Upstairs UUID: b7ea5033-7428-4d24-baa0-ce80fa7cdad9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561804392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Crucible stats registered with UUID: b7ea5033-7428-4d24-baa0-ce80fa7cdad9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.561820001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:03.5618334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562023657Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562059195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562074754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562157128Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562175487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"b7ea5033-7428-4d24-baa0-ce80fa7cdad9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562192266Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562205105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562219024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562232753Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562246532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562261011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.56227525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35047)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562324987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36110)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562456638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55497)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.562483556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563037118Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563119852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563196047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563389984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563504606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[0]R flush_numbers[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563571232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.56358895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563605949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[1]R flush_numbers[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563621228Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563636427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563652646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[2]R flush_numbers[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563670545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563685434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563701913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563719771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.56373414Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.56374732Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563760789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"":"downstairs","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563774748Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563787217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"0","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563801066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"1","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2026-03-16T21:32:03.563814985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"client":"2","":"downstairs","session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563832734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563846233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"b7ea5033-7428-4d24-baa0-ce80fa7cdad9 is now active with session: 349371ec-e593-4007-a87c-e036ded4584b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563861132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563874001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216,"session_id":"349371ec-e593-4007-a87c-e036ded4584b"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.56388905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} {"msg":"Activated sub_volume b7ea5033-7428-4d24-baa0-ce80fa7cdad9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:03.563900899Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 284 len 20480 data: 3 3 3 3 3 02/30 Write block 181 len 8192 data: 2 2 03/30 Write block 165 len 12288 data: 2 2 2 04/30 Write block 487 len 12288 data: 2 2 2 05/30 Write block 380 len 32768 data: 2 2 2 2 2 2 2 2 06/30 Write block 358 len 4096 data: 3 07/30 Flush 08/30 Write block 29 len 24576 data: 2 2 2 2 2 2 09/30 Read block 19 len 32768 10/30 Write block 217 len 28672 data: 2 2 2 2 2 2 2 11/30 Read block 272 len 24576 12/30 Write block 499 len 20480 data: 2 2 2 2 2 13/30 Read block 333 len 16384 14/30 Write block 375 len 36864 data: 2 2 2 2 2 3 3 3 3 15/30 Write block 452 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Write block 209 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 181 len 12288 data: 3 3 2 18/30 Read block 251 len 40960 19/30 Write block 230 len 24576 data: 2 2 2 2 2 2 20/30 Write block 332 len 24576 data: 2 2 2 2 2 3 21/30 Write block 364 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Read block 319 len 32768 23/30 Write block 573 len 20480 data: 2 2 2 2 2 24/30 Read block 93 len 12288 25/30 Read block 542 len 8192 26/30 Read block 481 len 12288 27/30 Write block 478 len 24576 data: 2 2 2 2 3 3 28/30 Read block 379 len 4096 29/30 Read block 562 len 16384 30/30 Write block 57 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) None Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:03.575336238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 2 2 2 T F T <--- 2 040-059 2 2 2 2 2 2 T F T <--- 3 060-079 1 1 1 1 1 1 T F T <--- 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 1 4 3 1 3 F F F <--- 9 180-199 4 2 4 3 2 3 T F T <--- 10 200-219 2 2 2 2 2 2 T F T <--- 11 220-239 1 1 1 1 1 1 T F T <--- 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 4 2 4 3 2 3 F T F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 2 2 2 T F T <--- 17 340-359 4 2 4 3 2 3 F F F <--- 18 360-379 2 2 2 2 2 2 T F T <--- 19 380-399 4 1 4 3 1 3 T F T <--- 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 1 1 1 1 1 1 T F T <--- 23 460-479 1 1 1 1 1 1 T F T <--- 24 480-499 4 2 4 3 2 3 T F T <--- 25 500-519 2 2 2 2 2 2 T F T <--- 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 T F T <--- 29 580-599 2 2 2 2 2 2 F F F Max gen: 4, Max flush: 3 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 5 5 5 4 4 4 F F F 2 040-059 5 5 5 4 4 4 F F F 3 060-079 5 5 5 4 4 4 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 3 3 3 F F F 9 180-199 5 5 5 4 4 4 F F F 10 200-219 5 5 5 4 4 4 F F F 11 220-239 5 5 5 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 4 4 4 3 3 3 F F F 18 360-379 5 5 5 4 4 4 F F F 19 380-399 5 5 5 4 4 4 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 5 5 5 4 4 4 F F F 23 460-479 5 5 5 4 4 4 F F F 24 480-499 5 5 5 4 4 4 F F F 25 500-519 5 5 5 4 4 4 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 5 5 5 4 4 4 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 5, Max flush: 4 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.903054654Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.914777673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915135369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915173716Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915185535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915203184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"Crucible 843fbd4b-ca03-42e0-9325-1b70ed5d2e12 has session id: d9d7c114-cc6d-4bfa-b525-1535c1d14c43","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.91555584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Upstairs opts: Upstairs UUID: 843fbd4b-ca03-42e0-9325-1b70ed5d2e12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915590768Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Crucible stats registered with UUID: 843fbd4b-ca03-42e0-9325-1b70ed5d2e12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915606337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:07.915619646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915788434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915813363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915827712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915944434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.915961312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"843fbd4b-ca03-42e0-9325-1b70ed5d2e12 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.91599009Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.91600357Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916018069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916032378Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916045917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916060316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916074145Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:55986)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916089104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:62346)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916104053Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"ds_connection connected from Ok(127.0.0.1:33479)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916118962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916816834Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916845252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.916862031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917234555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917259254Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[0]R flush_numbers[0..12]: [1, 4, 4, 4, 2, 2, 1, 1, 3, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917273903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[0]R generation[0..12]: [1, 5, 5, 5, 2, 2, 1, 1, 4, 5, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917289812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917306441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[1]R flush_numbers[0..12]: [1, 4, 4, 4, 2, 2, 1, 1, 3, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917321729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[1]R generation[0..12]: [1, 5, 5, 5, 2, 2, 1, 1, 4, 5, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917336878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917353207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[2]R flush_numbers[0..12]: [1, 4, 4, 4, 2, 2, 1, 1, 3, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917368276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[2]R generation[0..12]: [1, 5, 5, 5, 2, 2, 1, 1, 4, 5, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917383015Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917399324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917412653Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Generation requested: 6 >= found:6"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917432302Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917445591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.91745879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"":"downstairs","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917475069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917487778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"0","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917501967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"1","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917515746Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"client":"2","":"downstairs","session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:32:07.917530175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917546504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"843fbd4b-ca03-42e0-9325-1b70ed5d2e12 is now active with session: d9d7c114-cc6d-4bfa-b525-1535c1d14c43","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917561623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917574372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224,"session_id":"d9d7c114-cc6d-4bfa-b525-1535c1d14c43"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917587541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} {"msg":"Activated sub_volume 843fbd4b-ca03-42e0-9325-1b70ed5d2e12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:07.917598971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 440 len 28672 data: 2 2 2 2 2 2 2 04/30 Write block 465 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Read block 235 len 16384 06/30 Read block 134 len 4096 07/30 Write block 463 len 36864 data: 2 2 3 3 3 3 3 3 3 08/30 Read block 131 len 20480 09/30 Read block 275 len 28672 10/30 Read block 119 len 16384 11/30 Write block 270 len 32768 data: 3 3 3 3 3 3 2 2 12/30 Flush 13/30 Write block 202 len 32768 data: 3 3 3 3 3 2 2 3 14/30 Write block 169 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 475 len 4096 16/30 Write block 392 len 12288 data: 2 2 2 17/30 Read block 293 len 16384 18/30 Read block 338 len 4096 19/30 Write block 27 len 24576 data: 2 2 3 3 3 3 20/30 Flush 21/30 Write block 242 len 12288 data: 2 2 2 22/30 Read block 457 len 24576 23/30 Read block 581 len 12288 24/30 Flush 25/30 Write block 52 len 4096 data: 2 26/30 Read block 64 len 36864 27/30 Write block 17 len 16384 data: 2 2 2 2 28/30 Write block 215 len 40960 data: 3 3 3 3 3 3 3 3 3 2 29/30 Read block 210 len 16384 30/30 Read block 408 len 24576 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:07.930376167Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T F <--- 1 020-039 6 6 5 7 7 4 T T F <--- 2 040-059 5 5 5 4 4 4 T T F <--- 3 060-079 6 6 5 5 5 4 F F F <--- 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 4 7 7 3 F F F <--- 9 180-199 5 5 5 4 4 4 F F F 10 200-219 6 6 5 7 7 4 T T F <--- 11 220-239 5 5 5 4 4 4 T T F <--- 12 240-259 6 6 1 8 8 1 F F F <--- 13 260-279 6 6 2 6 6 2 F F F <--- 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 4 4 4 3 3 3 F F F 18 360-379 5 5 5 4 4 4 F F F 19 380-399 6 6 5 7 7 4 F F F <--- 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 6 6 5 6 6 4 F F F <--- 23 460-479 6 6 5 6 6 4 F F F <--- 24 480-499 5 5 5 4 4 4 F F F 25 500-519 5 5 5 4 4 4 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 5 5 5 4 4 4 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 6, Max flush: 8 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 7 7 7 9 9 9 F F F 2 040-059 7 7 7 9 9 9 F F F 3 060-079 6 6 6 5 5 5 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 7 7 7 F F F 9 180-199 5 5 5 4 4 4 F F F 10 200-219 7 7 7 9 9 9 F F F 11 220-239 7 7 7 9 9 9 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 6 6 6 6 6 6 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 4 4 4 3 3 3 F F F 18 360-379 5 5 5 4 4 4 F F F 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 6 6 6 6 6 6 F F F 23 460-479 6 6 6 6 6 6 F F F 24 480-499 5 5 5 4 4 4 F F F 25 500-519 5 5 5 4 4 4 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 5 5 5 4 4 4 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 7, Max flush: 9 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.240567502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.252293271Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.252613769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.252665326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.252677495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.252695324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"Crucible e6232502-ff21-49de-80c4-9f913d8b1e84 has session id: 0a592e88-ffb4-43f7-b780-e4668382f56c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253002243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Upstairs opts: Upstairs UUID: e6232502-ff21-49de-80c4-9f913d8b1e84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25303408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Crucible stats registered with UUID: e6232502-ff21-49de-80c4-9f913d8b1e84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253049179Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:12.253062398Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253264295Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253300942Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253316341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253410575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253428243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"e6232502-ff21-49de-80c4-9f913d8b1e84 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253458401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25347232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"0","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253486159Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253500448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253514048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253528257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253542326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41097)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253597862Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37802)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25362119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58408)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.253636179Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25435235Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254380518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"0","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254459813Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254711596Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254749133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[0]R flush_numbers[0..12]: [9, 9, 9, 5, 2, 2, 1, 1, 7, 4, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254810589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[0]R generation[0..12]: [7, 7, 7, 6, 2, 2, 1, 1, 6, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254827388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254843987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[1]R flush_numbers[0..12]: [9, 9, 9, 5, 2, 2, 1, 1, 7, 4, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254858956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[1]R generation[0..12]: [7, 7, 7, 6, 2, 2, 1, 1, 6, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254874185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254890853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[2]R flush_numbers[0..12]: [9, 9, 9, 5, 2, 2, 1, 1, 7, 4, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254909562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[2]R generation[0..12]: [7, 7, 7, 6, 2, 2, 1, 1, 6, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254924901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25494158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254960989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.254975198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255000696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255014365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"":"downstairs","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255028464Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255041153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"0","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255057582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"1","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255072241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"client":"2","":"downstairs","session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.25508701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255100939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"e6232502-ff21-49de-80c4-9f913d8b1e84 is now active with session: 0a592e88-ffb4-43f7-b780-e4668382f56c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255115538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255128337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232,"session_id":"0a592e88-ffb4-43f7-b780-e4668382f56c"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255143216Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} {"msg":"Activated sub_volume e6232502-ff21-49de-80c4-9f913d8b1e84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:12.255154655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 364 len 40960 data: 3 3 3 3 3 3 3 3 3 3 02/30 Write block 127 len 16384 data: 2 2 2 2 03/30 Flush 04/30 Write block 38 len 4096 data: 3 05/30 Write block 72 len 16384 data: 3 3 3 3 06/30 Read block 178 len 36864 07/30 Read block 9 len 32768 08/30 Write block 525 len 24576 data: 2 2 2 2 2 2 09/30 Write block 32 len 36864 data: 4 3 3 3 3 3 4 3 3 10/30 Read block 25 len 20480 11/30 Write block 318 len 4096 data: 2 12/30 Write block 503 len 36864 data: 3 2 2 2 2 2 2 2 2 13/30 Write block 346 len 20480 data: 2 2 2 2 2 14/30 Write block 490 len 24576 data: 2 2 2 2 2 2 15/30 Write block 540 len 24576 data: 2 2 2 2 2 2 16/30 Read block 553 len 4096 17/30 Read block 586 len 16384 18/30 Write block 182 len 16384 data: 4 3 2 2 19/30 Read block 536 len 32768 20/30 Read block 130 len 32768 21/30 Read block 26 len 24576 22/30 Read block 249 len 36864 23/30 Read block 554 len 32768 24/30 Write block 17 len 24576 data: 3 3 3 3 2 2 25/30 Write block 443 len 4096 data: 3 26/30 Write block 369 len 16384 data: 4 4 4 4 27/30 Read block 548 len 24576 28/30 Read block 443 len 8192 29/30 Write block 162 len 32768 data: 2 2 2 3 3 3 2 3 30/30 Write block 553 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 25 29 Done 25 25 2 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:12.266679298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 T T F <--- 1 020-039 7 7 7 9 9 9 T T T 2 040-059 7 7 7 9 9 9 T T F <--- 3 060-079 6 6 6 5 5 5 T T T 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 8 8 8 10 10 10 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 7 7 7 T T F <--- 9 180-199 5 5 5 4 4 4 T T F <--- 10 200-219 7 7 7 9 9 9 F F F 11 220-239 7 7 7 9 9 9 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 6 6 6 6 6 6 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 T T F <--- 16 320-339 5 5 5 4 4 4 F F F 17 340-359 4 4 4 3 3 3 T T F <--- 18 360-379 8 8 8 10 10 10 T T F <--- 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 6 6 6 6 6 6 T T F <--- 23 460-479 6 6 6 6 6 6 F F F 24 480-499 5 5 5 4 4 4 T T F <--- 25 500-519 5 5 5 4 4 4 T T F <--- 26 520-539 2 2 2 2 2 2 T T F <--- 27 540-559 1 1 1 1 1 1 T T F <--- 28 560-579 5 5 5 4 4 4 T T F <--- 29 580-599 2 2 2 2 2 2 F F F Max gen: 8, Max flush: 10 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 11 11 11 F F F 1 020-039 9 9 9 11 11 11 F F F 2 040-059 9 9 9 11 11 11 F F F 3 060-079 9 9 9 11 11 11 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 8 8 8 10 10 10 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 9 9 9 11 11 11 F F F 9 180-199 9 9 9 11 11 11 F F F 10 200-219 7 7 7 9 9 9 F F F 11 220-239 7 7 7 9 9 9 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 6 6 6 6 6 6 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 9 9 9 11 11 11 F F F 18 360-379 9 9 9 11 11 11 F F F 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 9 9 9 11 11 11 F F F 23 460-479 6 6 6 6 6 6 F F F 24 480-499 9 9 9 11 11 11 F F F 25 500-519 9 9 9 11 11 11 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F F F 28 560-579 9 9 9 11 11 11 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 9, Max flush: 11 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.592677853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:16.603368542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.603426208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.61379323Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614091639Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614130487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614142606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614160545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"Crucible 11b10689-da92-4602-93af-14de35ca8292 has session id: da0c5349-d36b-4a9b-b25e-454741a711d5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614496812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Upstairs opts: Upstairs UUID: 11b10689-da92-4602-93af-14de35ca8292, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614563317Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Crucible stats registered with UUID: 11b10689-da92-4602-93af-14de35ca8292","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614580656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:16.614594335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614756804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614781772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614799841Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614924343Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614943061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"11b10689-da92-4602-93af-14de35ca8292 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614973009Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.614987018Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615001197Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615015156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615028775Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615051484Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615065713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44002)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615080732Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48790)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615095591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48549)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615235041Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615827541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615850759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.615929414Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616203325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616242193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[0]R flush_numbers[0..12]: [11, 11, 11, 11, 2, 2, 10, 1, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616313038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[0]R generation[0..12]: [9, 9, 9, 9, 2, 2, 8, 1, 9, 9, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616329327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616345885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[1]R flush_numbers[0..12]: [11, 11, 11, 11, 2, 2, 10, 1, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616361564Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[1]R generation[0..12]: [9, 9, 9, 9, 2, 2, 8, 1, 9, 9, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616377213Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616394082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[2]R flush_numbers[0..12]: [11, 11, 11, 11, 2, 2, 10, 1, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616409551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[2]R generation[0..12]: [9, 9, 9, 9, 2, 2, 8, 1, 9, 9, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.61642429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616440599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616454268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616474547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616487826Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616503055Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"":"downstairs","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616517224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616529923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"0","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616543872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"1","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616557791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"client":"2","":"downstairs","session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.61657501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616589149Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"11b10689-da92-4602-93af-14de35ca8292 is now active with session: da0c5349-d36b-4a9b-b25e-454741a711d5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616604168Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616616857Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240,"session_id":"da0c5349-d36b-4a9b-b25e-454741a711d5"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616630126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} {"msg":"Activated sub_volume 11b10689-da92-4602-93af-14de35ca8292","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:16.616641545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 32768 data: 2 2 3 3 3 3 3 3 02/30 Read block 304 len 4096 03/30 Read block 489 len 12288 04/30 Write block 138 len 20480 data: 2 2 2 2 2 05/30 Write block 351 len 40960 data: 2 2 2 3 3 3 3 4 3 3 06/30 Flush 07/30 Write block 507 len 16384 data: 3 3 3 3 08/30 Read block 375 len 28672 09/30 Read block 198 len 4096 10/30 Write block 498 len 40960 data: 2 3 3 3 3 4 3 3 3 4 11/30 Write block 1 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Read block 284 len 12288 13/30 Write block 216 len 8192 data: 4 4 14/30 Write block 68 len 36864 data: 2 2 3 3 4 4 4 4 3 15/30 Read block 155 len 24576 16/30 Flush 17/30 Write block 244 len 28672 data: 3 2 2 2 2 2 2 18/30 Read block 366 len 12288 19/30 Read block 135 len 8192 20/30 Read block 437 len 36864 21/30 Write block 419 len 40960 data: 3 3 3 3 3 3 3 3 3 3 22/30 Flush 23/30 Write block 555 len 40960 data: 3 3 3 3 3 3 3 2 2 2 24/30 Read block 449 len 40960 25/30 Write block 417 len 16384 data: 3 3 4 4 26/30 Write block 24 len 4096 data: 2 27/30 Read block 554 len 28672 28/30 Write block 395 len 36864 data: 2 2 2 2 2 2 2 2 2 29/30 Write block 367 len 8192 data: 4 4 30/30 Read block 38 len 24576 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:16.630440782Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 10 10 11 13 13 F F F <--- 1 020-039 9 9 9 11 11 11 F T T <--- 2 040-059 9 9 9 11 11 11 F F F 3 060-079 9 10 10 11 13 13 F F F <--- 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 8 10 10 10 12 12 F F F <--- 7 140-159 1 10 10 1 12 12 F F F <--- 8 160-179 9 9 9 11 11 11 F F F 9 180-199 9 9 9 11 11 11 F F F 10 200-219 7 10 10 9 13 13 F F F <--- 11 220-239 7 7 7 9 9 9 F F F 12 240-259 6 10 10 8 14 14 F F F <--- 13 260-279 6 6 6 6 6 6 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 9 10 10 11 12 12 F F F <--- 18 360-379 9 10 10 11 12 12 F T T <--- 19 380-399 6 6 6 7 7 7 F T T <--- 20 400-419 2 10 10 2 14 14 F T T <--- 21 420-439 2 10 10 2 14 14 F T T <--- 22 440-459 9 10 10 11 12 12 T F F <--- 23 460-479 6 6 6 6 6 6 F F F 24 480-499 9 10 10 11 13 13 F F F <--- 25 500-519 9 10 10 11 13 13 F F F <--- 26 520-539 9 9 9 11 11 11 F F F 27 540-559 9 9 9 11 11 11 F T T <--- 28 560-579 9 9 9 11 11 11 F T T <--- 29 580-599 2 2 2 2 2 2 F F F Max gen: 10, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 11 11 11 15 15 15 F F F 2 040-059 9 9 9 11 11 11 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 10 10 10 12 12 12 F F F 8 160-179 9 9 9 11 11 11 F F F 9 180-199 9 9 9 11 11 11 F F F 10 200-219 10 10 10 13 13 13 F F F 11 220-239 7 7 7 9 9 9 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 6 6 6 6 6 6 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 9 9 9 11 11 11 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 10 10 10 12 12 12 F F F 18 360-379 11 11 11 15 15 15 F F F 19 380-399 11 11 11 15 15 15 F F F 20 400-419 11 11 11 15 15 15 F F F 21 420-439 11 11 11 15 15 15 F F F 22 440-459 10 10 10 12 12 12 F F F 23 460-479 6 6 6 6 6 6 F F F 24 480-499 10 10 10 13 13 13 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 15 15 15 F F F 28 560-579 11 11 11 15 15 15 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 11, Max flush: 15 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.964954205Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.976791246Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977112884Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977154001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97716642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977187889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"Crucible 66aa0d22-e9c1-40a0-8e19-a376b30d1587 has session id: dc8a3c70-1b23-4de3-a611-77f8b2172036","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977529215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Upstairs opts: Upstairs UUID: 66aa0d22-e9c1-40a0-8e19-a376b30d1587, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977563803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Crucible stats registered with UUID: 66aa0d22-e9c1-40a0-8e19-a376b30d1587","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977579442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:20.977593171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977762799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977787688Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977802207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977928878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977946187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"66aa0d22-e9c1-40a0-8e19-a376b30d1587 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977965346Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977978335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.977992724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978006873Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978020562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978037361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97805182Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"ds_connection connected from Ok(127.0.0.1:48590)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978113076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"ds_connection connected from Ok(127.0.0.1:57914)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97819242Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"ds_connection connected from Ok(127.0.0.1:48029)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978210589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978894752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97892489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.978949588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979256467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979303494Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979339662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[0]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 9, 9, 10, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97935823Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979374769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979390108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[1]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 9, 9, 10, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979405007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979423956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979440105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[2]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 9, 9, 10, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979455444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979471763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979492221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97950634Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979519899Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979534308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"":"downstairs","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979548837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979561707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"0","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979575786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 client":"1","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979592804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"client":"2","":"downstairs","session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979607013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979620123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"66aa0d22-e9c1-40a0-8e19-a376b30d1587 is now active with session: dc8a3c70-1b23-4de3-a611-77f8b2172036","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979634992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979649131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248,"session_id":"dc8a3c70-1b23-4de3-a611-77f8b2172036"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.97966277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} {"msg":"Activated sub_volume 66aa0d22-e9c1-40a0-8e19-a376b30d1587","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:20.979674279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 302 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 522 len 16384 03/30 Read block 398 len 24576 04/30 Read block 478 len 8192 05/30 Flush 06/30 Write block 416 len 8192 data: 3 4 07/30 Write block 407 len 40960 data: 2 2 2 2 2 2 2 3 3 4 08/30 Read block 435 len 32768 09/30 Read block 577 len 40960 10/30 Read block 445 len 20480 11/30 Read block 308 len 20480 12/30 Write block 196 len 20480 data: 2 2 3 3 3 13/30 Read block 84 len 16384 14/30 Read block 292 len 20480 15/30 Write block 192 len 32768 data: 2 2 2 2 3 3 4 4 16/30 Write block 444 len 20480 data: 3 3 3 2 2 17/30 Read block 337 len 8192 18/30 Write block 460 len 4096 data: 2 19/30 Flush 20/30 Write block 175 len 32768 data: 3 3 3 3 2 2 4 5 21/30 Read block 485 len 40960 22/30 Write block 275 len 32768 data: 4 3 3 2 2 2 2 2 23/30 Read block 195 len 28672 24/30 Read block 541 len 20480 25/30 Write block 411 len 24576 data: 3 3 3 4 4 5 26/30 Write block 572 len 4096 data: 2 27/30 Read block 266 len 40960 28/30 Read block 438 len 40960 29/30 Write block 360 len 32768 data: 4 4 3 2 4 4 4 5 30/30 Read block 87 len 40960 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:20.992996459Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 11 11 11 15 15 15 F F F 2 040-059 9 9 9 11 11 11 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 10 10 10 12 12 12 F F F 8 160-179 9 9 9 11 11 11 T F T <--- 9 180-199 12 9 12 17 11 17 T F T <--- 10 200-219 12 10 12 17 13 17 F F F <--- 11 220-239 7 7 7 9 9 9 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 6 6 6 6 6 6 T F T <--- 14 280-299 4 4 4 3 3 3 T F T <--- 15 300-319 12 9 12 16 11 16 F T F <--- 16 320-339 5 5 5 4 4 4 F F F 17 340-359 10 10 10 12 12 12 F F F 18 360-379 11 11 11 15 15 15 T F T <--- 19 380-399 11 11 11 15 15 15 F F F 20 400-419 12 11 12 17 15 17 T F T <--- 21 420-439 11 11 11 15 15 15 F F F 22 440-459 12 10 12 17 12 17 F F F <--- 23 460-479 12 6 12 17 6 17 F F F <--- 24 480-499 10 10 10 13 13 13 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 15 15 15 F F F 28 560-579 11 11 11 15 15 15 T F T <--- 29 580-599 2 2 2 2 2 2 F F F Max gen: 12, Max flush: 17 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 11 11 11 15 15 15 F F F 2 040-059 9 9 9 11 11 11 F F F 3 060-079 10 10 10 13 13 13 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 10 10 10 12 12 12 F F F 8 160-179 13 13 13 18 18 18 F F F 9 180-199 13 13 13 18 18 18 F F F 10 200-219 12 12 12 17 17 17 F F F 11 220-239 7 7 7 9 9 9 F F F 12 240-259 10 10 10 14 14 14 F F F 13 260-279 13 13 13 18 18 18 F F F 14 280-299 13 13 13 18 18 18 F F F 15 300-319 12 12 12 16 16 16 F F F 16 320-339 5 5 5 4 4 4 F F F 17 340-359 10 10 10 12 12 12 F F F 18 360-379 13 13 13 18 18 18 F F F 19 380-399 11 11 11 15 15 15 F F F 20 400-419 13 13 13 18 18 18 F F F 21 420-439 11 11 11 15 15 15 F F F 22 440-459 12 12 12 17 17 17 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 10 10 10 13 13 13 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 11 11 15 15 15 F F F 28 560-579 13 13 13 18 18 18 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 13, Max flush: 18 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.305372214Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:25.316092342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.316146538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.326591334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.326902053Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32694007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32695241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.326970038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"Crucible 802d5ac1-3f6c-49ef-a835-5695095fe7bb has session id: 4e001604-44ef-4088-9427-41f3d183daa2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327295426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Upstairs opts: Upstairs UUID: 802d5ac1-3f6c-49ef-a835-5695095fe7bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327329714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Crucible stats registered with UUID: 802d5ac1-3f6c-49ef-a835-5695095fe7bb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327345763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:25.327359152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32753803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327574057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327590056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327703618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327720767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"802d5ac1-3f6c-49ef-a835-5695095fe7bb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327738226Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327750825Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327764714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327778553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"2","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327792102Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327806421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.3278207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54980)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327874667Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58667)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327902795Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49326)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.327919084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.328624265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.328666352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.328722469Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"2","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.328978301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329036237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[0]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 18, 18, 17, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329103533Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[0]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 13, 13, 12, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329120471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32913655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[1]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 18, 18, 17, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329151809Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[1]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 13, 13, 12, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329166778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329182977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[2]R flush_numbers[0..12]: [13, 15, 11, 13, 2, 2, 12, 12, 18, 18, 17, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329197966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[2]R generation[0..12]: [10, 11, 9, 10, 2, 2, 10, 10, 13, 13, 12, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329213225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329229594Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-16T21:32:25.329242833Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329264252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329277411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32929097Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"":"downstairs","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329305329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329317618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"0","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329331867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"1","":"downstairs","session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329345956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"client":"2","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329363285Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329376904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"802d5ac1-3f6c-49ef-a835-5695095fe7bb is now active with session: 4e001604-44ef-4088-9427-41f3d183daa2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329391903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329404532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256,"session_id":"4e001604-44ef-4088-9427-41f3d183daa2"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.329417701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} {"msg":"Activated sub_volume 802d5ac1-3f6c-49ef-a835-5695095fe7bb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:25.32942915Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 259 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 164 len 32768 03/30 Write block 543 len 8192 data: 3 3 04/30 Read block 588 len 4096 05/30 Read block 292 len 32768 06/30 Write block 63 len 12288 data: 2 2 2 07/30 Write block 172 len 40960 data: 3 3 3 4 4 4 4 3 3 5 08/30 Write block 49 len 8192 data: 2 2 09/30 Write block 402 len 40960 data: 3 3 2 2 2 3 3 3 3 4 10/30 Read block 346 len 24576 11/30 Write block 362 len 8192 data: 4 3 12/30 Write block 23 len 12288 data: 2 3 2 13/30 Read block 415 len 24576 14/30 Flush 15/30 Write block 222 len 40960 data: 4 4 3 2 2 2 2 2 3 3 16/30 Read block 522 len 4096 17/30 Write block 449 len 24576 data: 2 3 3 4 4 4 18/30 Write block 197 len 32768 data: 4 5 5 4 3 4 4 4 19/30 Write block 160 len 28672 data: 2 2 3 3 3 4 4 20/30 Read block 333 len 16384 21/30 Read block 469 len 32768 22/30 Write block 373 len 24576 data: 4 2 3 3 3 3 23/30 Read block 580 len 20480 24/30 Read block 569 len 20480 25/30 Read block 299 len 28672 26/30 Read block 489 len 40960 27/30 Write block 39 len 40960 data: 4 4 2 2 2 2 2 2 2 2 28/30 Read block 195 len 8192 29/30 Read block 270 len 20480 30/30 Write block 322 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:25.342329559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 11 14 14 15 19 19 F T T <--- 2 040-059 9 14 14 11 19 19 F T T <--- 3 060-079 10 14 14 13 19 19 F F F <--- 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 10 10 10 12 12 12 F F F 8 160-179 13 14 14 18 19 19 F T T <--- 9 180-199 13 14 14 18 19 19 F T T <--- 10 200-219 12 12 12 17 17 17 F T T <--- 11 220-239 7 7 7 9 9 9 F T T <--- 12 240-259 10 14 14 14 19 19 T F F <--- 13 260-279 13 14 14 18 19 19 T F F <--- 14 280-299 13 13 13 18 18 18 F F F 15 300-319 12 12 12 16 16 16 F F F 16 320-339 5 5 5 4 4 4 F T T <--- 17 340-359 10 10 10 12 12 12 F F F 18 360-379 13 14 14 18 19 19 F T T <--- 19 380-399 11 11 11 15 15 15 F F F 20 400-419 13 14 14 18 19 19 F F F <--- 21 420-439 11 11 11 15 15 15 F F F 22 440-459 12 12 12 17 17 17 F T T <--- 23 460-479 12 12 12 17 17 17 F F F 24 480-499 10 10 10 13 13 13 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 11 14 14 15 19 19 F F F <--- 28 560-579 13 13 13 18 18 18 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 14, Max flush: 19 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 13 13 13 F F F 1 020-039 15 15 15 20 20 20 F F F 2 040-059 15 15 15 20 20 20 F F F 3 060-079 14 14 14 19 19 19 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 10 10 10 12 12 12 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 15 15 15 20 20 20 F F F 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 19 19 19 F F F 13 260-279 14 14 14 19 19 19 F F F 14 280-299 13 13 13 18 18 18 F F F 15 300-319 12 12 12 16 16 16 F F F 16 320-339 15 15 15 20 20 20 F F F 17 340-359 10 10 10 12 12 12 F F F 18 360-379 15 15 15 20 20 20 F F F 19 380-399 11 11 11 15 15 15 F F F 20 400-419 14 14 14 19 19 19 F F F 21 420-439 11 11 11 15 15 15 F F F 22 440-459 15 15 15 20 20 20 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 10 10 10 13 13 13 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 14 14 14 19 19 19 F F F 28 560-579 13 13 13 18 18 18 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 15, Max flush: 20 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.673709026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.6854962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.685807629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.685843887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.685859276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.685877174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"Crucible 56bff23e-aabb-404e-a6a7-0f0771470a3c has session id: 3ddf4016-44af-4f2c-ba59-26b90ac9b047","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686216071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Upstairs opts: Upstairs UUID: 56bff23e-aabb-404e-a6a7-0f0771470a3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686250369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Crucible stats registered with UUID: 56bff23e-aabb-404e-a6a7-0f0771470a3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686266028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:29.686279397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686457745Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686484063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686498812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686622853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686641322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"56bff23e-aabb-404e-a6a7-0f0771470a3c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686656081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.68666898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686683149Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686697358Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686713327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686727776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686742005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"ds_connection connected from Ok(127.0.0.1:47566)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686774243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"ds_connection connected from Ok(127.0.0.1:53738)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686789352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"ds_connection connected from Ok(127.0.0.1:34586)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.686804181Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.687639004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.687672482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.687688861Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688013878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688046656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[0]R flush_numbers[0..12]: [13, 20, 20, 19, 2, 2, 12, 12, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688105232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[0]R generation[0..12]: [10, 15, 15, 14, 2, 2, 10, 10, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688123101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.6881399Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[1]R flush_numbers[0..12]: [13, 20, 20, 19, 2, 2, 12, 12, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688158499Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[1]R generation[0..12]: [10, 15, 15, 14, 2, 2, 10, 10, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688174257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688190506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[2]R flush_numbers[0..12]: [13, 20, 20, 19, 2, 2, 12, 12, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688205765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[2]R generation[0..12]: [10, 15, 15, 14, 2, 2, 10, 10, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688220814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688237133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Max found gen is 16","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2026-03-16T21:32:29.688256172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688270521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.68828369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688297289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"":"downstairs","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688311238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688324087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"0","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688338286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"1","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688352115Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"client":"2","":"downstairs","session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688369624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688384723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"56bff23e-aabb-404e-a6a7-0f0771470a3c is now active with session: 3ddf4016-44af-4f2c-ba59-26b90ac9b047","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688400042Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688412911Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264,"session_id":"3ddf4016-44af-4f2c-ba59-26b90ac9b047"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.68842617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} {"msg":"Activated sub_volume 56bff23e-aabb-404e-a6a7-0f0771470a3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:29.688437549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 4096 data: 4 02/30 Write block 148 len 24576 data: 2 2 2 2 2 2 03/30 Flush 04/30 Write block 359 len 20480 data: 4 5 5 5 4 05/30 Flush 06/30 Write block 419 len 12288 data: 5 5 4 07/30 Read block 508 len 20480 08/30 Write block 389 len 4096 data: 2 09/30 Write block 278 len 20480 data: 3 3 3 3 3 10/30 Write block 558 len 28672 data: 4 4 4 4 3 3 3 11/30 Read block 0 len 36864 12/30 Flush 13/30 Write block 437 len 28672 data: 3 3 2 3 3 3 4 14/30 Read block 2 len 36864 15/30 Write block 17 len 28672 data: 4 4 4 4 3 3 3 16/30 Read block 433 len 12288 17/30 Read block 579 len 8192 18/30 Write block 496 len 36864 data: 2 2 3 4 4 4 4 5 4 19/30 Read block 58 len 36864 20/30 Flush 21/30 Write block 387 len 40960 data: 3 2 3 2 2 3 3 3 3 3 22/30 Read block 363 len 40960 23/30 Write block 304 len 24576 data: 3 3 3 3 3 3 24/30 Read block 74 len 28672 25/30 Write block 336 len 24576 data: 3 4 3 3 2 2 26/30 Flush 27/30 Write block 23 len 8192 data: 4 4 28/30 Write block 144 len 8192 data: 2 2 29/30 Read block 174 len 8192 30/30 Write block 309 len 16384 data: 4 3 3 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:29.700742459Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 10 16 24 13 24 F F F <--- 1 020-039 16 15 16 24 20 24 T F T <--- 2 040-059 15 15 15 20 20 20 F F F 3 060-079 14 14 14 19 19 19 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 16 10 16 21 12 21 T F T <--- 8 160-179 15 15 15 20 20 20 F F F 9 180-199 15 15 15 20 20 20 F F F 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 19 19 19 F F F 13 260-279 16 14 16 23 19 23 F F F <--- 14 280-299 16 13 16 23 18 23 F F F <--- 15 300-319 16 12 16 25 16 25 T F T <--- 16 320-339 16 15 16 25 20 25 F F F <--- 17 340-359 16 10 16 25 12 25 F F F <--- 18 360-379 16 15 16 22 20 22 F F F <--- 19 380-399 16 11 16 25 15 25 F F F <--- 20 400-419 16 14 16 23 19 23 F T F <--- 21 420-439 16 11 16 24 15 24 F F F <--- 22 440-459 16 15 16 24 20 24 F F F <--- 23 460-479 12 12 12 17 17 17 F F F 24 480-499 16 10 16 24 13 24 F F F <--- 25 500-519 16 10 16 24 13 24 F F F <--- 26 520-539 9 9 9 11 11 11 F F F 27 540-559 16 14 16 23 19 23 F F F <--- 28 560-579 16 13 16 23 18 23 F F F <--- 29 580-599 2 2 2 2 2 2 F F F Max gen: 16, Max flush: 25 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 15 15 15 20 20 20 F F F 3 060-079 14 14 14 19 19 19 F F F 4 080-099 2 2 2 2 2 2 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 17 17 17 26 26 26 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 15 15 15 20 20 20 F F F 10 200-219 15 15 15 20 20 20 F F F 11 220-239 15 15 15 20 20 20 F F F 12 240-259 14 14 14 19 19 19 F F F 13 260-279 16 16 16 23 23 23 F F F 14 280-299 16 16 16 23 23 23 F F F 15 300-319 17 17 17 26 26 26 F F F 16 320-339 16 16 16 25 25 25 F F F 17 340-359 16 16 16 25 25 25 F F F 18 360-379 16 16 16 22 22 22 F F F 19 380-399 16 16 16 25 25 25 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 16 16 16 24 24 24 F F F 25 500-519 16 16 16 24 24 24 F F F 26 520-539 9 9 9 11 11 11 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 17, Max flush: 26 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.025478309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.037183429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.037550574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.037589032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.037601471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.03761907Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"Crucible 1016ab8b-70e5-4234-baa6-7677d1f7ba3c has session id: e03ec652-f605-4e47-b994-635fee0688bb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.037991754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Upstairs opts: Upstairs UUID: 1016ab8b-70e5-4234-baa6-7677d1f7ba3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038026082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Crucible stats registered with UUID: 1016ab8b-70e5-4234-baa6-7677d1f7ba3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038041721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:34.03805511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038236797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038261736Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038276305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038403956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038422545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"1016ab8b-70e5-4234-baa6-7677d1f7ba3c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038439374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038452123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038466052Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038480121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.03849356Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038507679Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038521958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56000)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038544846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56381)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038559985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:60466)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.038574894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039378189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039417107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039434556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039735385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039751924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[0]R flush_numbers[0..12]: [24, 26, 20, 19, 2, 2, 12, 26, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039785512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[0]R generation[0..12]: [16, 17, 15, 14, 2, 2, 10, 17, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.03980214Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039818789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[1]R flush_numbers[0..12]: [24, 26, 20, 19, 2, 2, 12, 26, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039834658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[1]R generation[0..12]: [16, 17, 15, 14, 2, 2, 10, 17, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039849967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039866386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[2]R flush_numbers[0..12]: [24, 26, 20, 19, 2, 2, 12, 26, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039881585Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[2]R generation[0..12]: [16, 17, 15, 14, 2, 2, 10, 17, 15, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039896754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039912993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039926442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:32:34.039943201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039962489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039975929Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"":"downstairs","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.039990198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040003127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"0","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040017036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"1","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040030855Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"client":"2","":"downstairs","session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040045354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040058843Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"1016ab8b-70e5-4234-baa6-7677d1f7ba3c is now active with session: e03ec652-f605-4e47-b994-635fee0688bb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040076652Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040089671Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272,"session_id":"e03ec652-f605-4e47-b994-635fee0688bb"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.04010327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} {"msg":"Activated sub_volume 1016ab8b-70e5-4234-baa6-7677d1f7ba3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:34.040115069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 20480 data: 2 2 2 2 2 02/30 Write block 503 len 28672 data: 6 5 4 4 5 4 4 03/30 Read block 140 len 36864 04/30 Write block 288 len 20480 data: 4 2 2 2 2 05/30 Flush 06/30 Write block 257 len 40960 data: 2 2 3 3 3 3 3 3 3 2 07/30 Write block 402 len 32768 data: 4 4 3 3 3 4 5 4 08/30 Read block 161 len 40960 09/30 Write block 515 len 16384 data: 3 3 3 3 10/30 Write block 530 len 20480 data: 3 2 2 2 2 11/30 Read block 400 len 24576 12/30 Read block 560 len 4096 13/30 Read block 59 len 40960 14/30 Read block 252 len 12288 15/30 Read block 416 len 32768 16/30 Write block 489 len 28672 data: 3 3 3 3 3 3 3 17/30 Read block 264 len 40960 18/30 Write block 63 len 36864 data: 3 3 3 2 2 3 3 4 4 19/30 Read block 460 len 8192 20/30 Write block 225 len 32768 data: 3 3 3 3 3 4 4 3 21/30 Read block 256 len 32768 22/30 Read block 191 len 16384 23/30 Read block 425 len 32768 24/30 Write block 192 len 36864 data: 3 3 3 3 4 5 6 6 5 25/30 Read block 505 len 36864 26/30 Flush 27/30 Write block 242 len 16384 data: 3 3 4 3 28/30 Write block 48 len 20480 data: 3 3 3 2 3 29/30 Read block 145 len 28672 30/30 Read block 407 len 4096 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 7 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:34.053345865Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 15 15 15 20 20 20 T T F <--- 3 060-079 18 18 14 28 28 19 F F F <--- 4 080-099 18 18 2 27 27 2 F F T <--- 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 17 17 17 26 26 26 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 18 18 15 28 28 20 F F F <--- 10 200-219 18 18 15 28 28 20 F F F <--- 11 220-239 18 18 15 28 28 20 F F F <--- 12 240-259 18 18 14 28 28 19 T T F <--- 13 260-279 18 18 16 28 28 23 F F F <--- 14 280-299 18 18 16 27 27 23 F F F <--- 15 300-319 17 17 17 26 26 26 F F F 16 320-339 16 16 16 25 25 25 F F F 17 340-359 16 16 16 25 25 25 F F F 18 360-379 16 16 16 22 22 22 F F F 19 380-399 16 16 16 25 25 25 F F F 20 400-419 18 18 16 28 28 23 F F F <--- 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 18 18 16 28 28 24 F F F <--- 25 500-519 18 18 16 28 28 24 F F F <--- 26 520-539 18 18 9 28 28 11 F F F <--- 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 18, Max flush: 28 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 19 19 19 29 29 29 F F F 3 060-079 18 18 18 28 28 28 F F F 4 080-099 18 18 18 27 27 27 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 F F F 7 140-159 17 17 17 26 26 26 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 18 18 18 28 28 28 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F F F 12 240-259 19 19 19 29 29 29 F F F 13 260-279 18 18 18 28 28 28 F F F 14 280-299 18 18 18 27 27 27 F F F 15 300-319 17 17 17 26 26 26 F F F 16 320-339 16 16 16 25 25 25 F F F 17 340-359 16 16 16 25 25 25 F F F 18 360-379 16 16 16 22 22 22 F F F 19 380-399 16 16 16 25 25 25 F F F 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F F F 23 460-479 12 12 12 17 17 17 F F F 24 480-499 18 18 18 28 28 28 F F F 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 19, Max flush: 29 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.367535277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379172821Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379500439Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379540066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379556155Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379574364Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"Crucible f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9 has session id: 99f64810-227b-4a2a-b0d0-0ded2708ef23","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379899492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Upstairs opts: Upstairs UUID: f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379933989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Crucible stats registered with UUID: f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.379949618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:38.379962627Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380151685Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380177003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380191462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380309954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380331222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380347151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.38035989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380374159Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380388518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380404697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380419336Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380433405Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"ds_connection connected from Ok(127.0.0.1:64720)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380543048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"ds_connection connected from Ok(127.0.0.1:58102)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380561237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"ds_connection connected from Ok(127.0.0.1:61678)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.380576316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.38123695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381266408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381292407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381616454Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381630793Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[0]R flush_numbers[0..12]: [24, 26, 29, 28, 27, 2, 12, 26, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381666261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[0]R generation[0..12]: [16, 17, 19, 18, 18, 2, 10, 17, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.3816829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381700039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[1]R flush_numbers[0..12]: [24, 26, 29, 28, 27, 2, 12, 26, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381718427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[1]R generation[0..12]: [16, 17, 19, 18, 18, 2, 10, 17, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381733966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381750305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[2]R flush_numbers[0..12]: [24, 26, 29, 28, 27, 2, 12, 26, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381765724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[2]R generation[0..12]: [16, 17, 19, 18, 18, 2, 10, 17, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381781163Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381797622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381810791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.3818306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381843729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381857298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"":"downstairs","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381871507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381884356Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"0","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381898355Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"1","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381911994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"client":"2","":"downstairs","session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:32:38.381926183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381942302Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9 is now active with session: 99f64810-227b-4a2a-b0d0-0ded2708ef23","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381957311Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.38197015Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280,"session_id":"99f64810-227b-4a2a-b0d0-0ded2708ef23"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381983789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} {"msg":"Activated sub_volume f1cabe4d-7ee0-45cd-890f-bbe8fcff71d9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:38.381995669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 32768 data: 3 3 3 3 2 2 2 2 02/30 Write block 49 len 20480 data: 4 4 3 4 2 03/30 Read block 127 len 16384 04/30 Write block 144 len 8192 data: 3 3 05/30 Read block 159 len 20480 06/30 Read block 84 len 8192 07/30 Write block 355 len 8192 data: 4 4 08/30 Read block 41 len 32768 09/30 Read block 427 len 8192 10/30 Flush 11/30 Write block 241 len 12288 data: 2 4 4 12/30 Read block 534 len 36864 13/30 Read block 292 len 28672 14/30 Read block 508 len 40960 15/30 Write block 324 len 8192 data: 3 3 16/30 Read block 372 len 28672 17/30 Read block 110 len 40960 18/30 Read block 170 len 20480 19/30 Read block 137 len 36864 20/30 Read block 506 len 20480 21/30 Write block 90 len 8192 data: 2 2 22/30 Write block 377 len 16384 data: 4 4 3 4 23/30 Write block 314 len 32768 data: 2 2 2 2 3 2 2 2 24/30 Flush 25/30 Write block 10 len 16384 data: 2 2 2 2 26/30 Read block 506 len 8192 27/30 Read block 213 len 40960 28/30 Write block 474 len 40960 data: 2 2 2 2 3 3 3 3 4 4 29/30 Write block 134 len 20480 data: 2 2 2 2 3 30/30 Read block 4 len 8192 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:38.393859558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 T T F <--- 1 020-039 17 17 17 26 26 26 F F F 2 040-059 20 20 19 30 30 29 F F F <--- 3 060-079 18 18 18 28 28 28 F F F 4 080-099 20 20 18 31 31 27 F F F <--- 5 100-119 2 2 2 2 2 2 F F F 6 120-139 10 10 10 12 12 12 T T F <--- 7 140-159 20 20 17 30 30 26 F F F <--- 8 160-179 15 15 15 20 20 20 F F F 9 180-199 18 18 18 28 28 28 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F F F 12 240-259 20 20 19 31 31 29 F F F <--- 13 260-279 18 18 18 28 28 28 F F F 14 280-299 20 20 18 30 30 27 F F F <--- 15 300-319 20 20 17 31 31 26 F F F <--- 16 320-339 20 20 16 31 31 25 F F F <--- 17 340-359 20 20 16 30 30 25 F F F <--- 18 360-379 20 20 16 31 31 22 F F F <--- 19 380-399 20 20 16 31 31 25 F F F <--- 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F F F 23 460-479 12 12 12 17 17 17 T T F <--- 24 480-499 18 18 18 28 28 28 T T F <--- 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 20, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 32 32 32 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 18 18 18 28 28 28 F F F 4 080-099 20 20 20 31 31 31 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 21 21 21 32 32 32 F F F 7 140-159 20 20 20 30 30 30 F F F 8 160-179 15 15 15 20 20 20 F F F 9 180-199 18 18 18 28 28 28 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F F F 12 240-259 20 20 20 31 31 31 F F F 13 260-279 18 18 18 28 28 28 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 20 20 20 31 31 31 F F F 16 320-339 20 20 20 31 31 31 F F F 17 340-359 20 20 20 30 30 30 F F F 18 360-379 20 20 20 31 31 31 F F F 19 380-399 20 20 20 31 31 31 F F F 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 21 21 21 32 32 32 F F F 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 21, Max flush: 32 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.711139568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:42.721816069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.721866765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.732293483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.7326258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.732663388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.732675747Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.732693875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"Crucible 750b1ee6-6e5d-475b-a0e5-bea356e4daa6 has session id: c534547d-ab37-4587-a44e-8a7e35a74e55","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733058001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Upstairs opts: Upstairs UUID: 750b1ee6-6e5d-475b-a0e5-bea356e4daa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733085329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Crucible stats registered with UUID: 750b1ee6-6e5d-475b-a0e5-bea356e4daa6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733099788Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:42.733116137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733259407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733284185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733299104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733434855Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733452884Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"750b1ee6-6e5d-475b-a0e5-bea356e4daa6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733475512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733488091Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.73350185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733515849Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733549257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733591784Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733617042Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:55430)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733633191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:44520)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.73364859Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:48837)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.733664529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734398289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734425637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734442616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734769994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734794562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[0]R flush_numbers[0..12]: [32, 26, 30, 28, 31, 2, 32, 30, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734809601Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[0]R generation[0..12]: [21, 17, 20, 18, 20, 2, 21, 20, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.73482556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734841969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[1]R flush_numbers[0..12]: [32, 26, 30, 28, 31, 2, 32, 30, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734857028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[1]R generation[0..12]: [21, 17, 20, 18, 20, 2, 21, 20, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734872397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734888606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[2]R flush_numbers[0..12]: [32, 26, 30, 28, 31, 2, 32, 30, 20, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734903954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[2]R generation[0..12]: [21, 17, 20, 18, 20, 2, 21, 20, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734919143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734935222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734952071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.73496616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.734992048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735010887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"":"downstairs","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735025066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735037915Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"0","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735051734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"1","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735065853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"client":"2","":"downstairs","session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735079952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735093202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"750b1ee6-6e5d-475b-a0e5-bea356e4daa6 is now active with session: c534547d-ab37-4587-a44e-8a7e35a74e55","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735108141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735123309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288,"session_id":"c534547d-ab37-4587-a44e-8a7e35a74e55"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735136909Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} {"msg":"Activated sub_volume 750b1ee6-6e5d-475b-a0e5-bea356e4daa6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:42.735148798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 111 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 126 len 8192 data: 2 3 03/30 Write block 265 len 4096 data: 4 04/30 Read block 246 len 24576 05/30 Read block 96 len 32768 06/30 Write block 322 len 12288 data: 3 3 4 07/30 Write block 595 len 8192 data: 2 2 08/30 Read block 188 len 24576 09/30 Flush 10/30 Write block 232 len 20480 data: 4 3 3 3 2 11/30 Write block 574 len 12288 data: 3 3 3 12/30 Read block 419 len 12288 13/30 Write block 388 len 12288 data: 3 4 3 14/30 Read block 202 len 32768 15/30 Read block 523 len 20480 16/30 Write block 252 len 12288 data: 2 2 2 17/30 Read block 561 len 36864 18/30 Read block 341 len 28672 19/30 Read block 535 len 24576 20/30 Read block 259 len 28672 21/30 Read block 448 len 36864 22/30 Read block 98 len 32768 23/30 Read block 289 len 16384 24/30 Write block 161 len 12288 data: 3 4 4 25/30 Read block 465 len 24576 26/30 Write block 301 len 32768 data: 2 3 3 4 4 4 4 4 27/30 Write block 14 len 12288 data: 2 2 2 28/30 Write block 224 len 24576 data: 4 4 4 4 4 4 29/30 Write block 157 len 28672 data: 2 2 2 3 4 5 5 30/30 Write block 445 len 16384 data: 4 4 3 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Write 1 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:42.74623663Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 32 32 32 F T T <--- 1 020-039 17 17 17 26 26 26 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 18 18 18 28 28 28 F F F 4 080-099 20 20 20 31 31 31 F F F 5 100-119 2 22 22 2 33 33 T F F <--- 6 120-139 21 22 22 32 33 33 F F F <--- 7 140-159 20 20 20 30 30 30 F T T <--- 8 160-179 15 15 15 20 20 20 F T T <--- 9 180-199 18 18 18 28 28 28 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 18 18 18 28 28 28 F T T <--- 12 240-259 20 20 20 31 31 31 F T T <--- 13 260-279 18 22 22 28 33 33 F F F <--- 14 280-299 20 20 20 30 30 30 F F F 15 300-319 20 20 20 31 31 31 F T T <--- 16 320-339 20 22 22 31 33 33 F F F <--- 17 340-359 20 20 20 30 30 30 F F F 18 360-379 20 20 20 31 31 31 F F F 19 380-399 20 20 20 31 31 31 F T T <--- 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 16 16 16 24 24 24 F T T <--- 23 460-479 21 21 21 32 32 32 F F F 24 480-499 21 21 21 32 32 32 F F F 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 16 16 16 23 23 23 F T T <--- 29 580-599 2 22 22 2 33 33 F F F <--- Max gen: 22, Max flush: 33 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 18 18 18 28 28 28 F F F 4 080-099 20 20 20 31 31 31 F F F 5 100-119 22 22 22 33 33 33 F F F 6 120-139 22 22 22 33 33 33 F F F 7 140-159 23 23 23 34 34 34 F F F 8 160-179 23 23 23 34 34 34 F F F 9 180-199 18 18 18 28 28 28 F F F 10 200-219 18 18 18 28 28 28 F F F 11 220-239 23 23 23 34 34 34 F F F 12 240-259 23 23 23 34 34 34 F F F 13 260-279 22 22 22 33 33 33 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 23 23 23 34 34 34 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 20 20 20 30 30 30 F F F 18 360-379 20 20 20 31 31 31 F F F 19 380-399 23 23 23 34 34 34 F F F 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 16 16 24 24 24 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 21 21 21 32 32 32 F F F 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 23 23 23 34 34 34 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 23, Max flush: 34 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.078767238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:47.089474066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.089524093Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.099787232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100081332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100116129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100128228Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100146137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"Crucible 54c1c4c3-37d7-4290-87e4-4a6ea87feda6 has session id: de78ced7-7c99-4f2d-8484-48406f612219","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100503053Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Upstairs opts: Upstairs UUID: 54c1c4c3-37d7-4290-87e4-4a6ea87feda6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.10053771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Crucible stats registered with UUID: 54c1c4c3-37d7-4290-87e4-4a6ea87feda6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100553469Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:47.100566868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100736397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100761495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100776074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100895536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100912245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"54c1c4c3-37d7-4290-87e4-4a6ea87feda6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100928574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100944073Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100958452Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100972471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.100996499Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101011118Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101025237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"ds_connection connected from Ok(127.0.0.1:36856)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101040096Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"ds_connection connected from Ok(127.0.0.1:64083)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101055065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"ds_connection connected from Ok(127.0.0.1:52548)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101171367Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101765136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101801604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.101937255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.10215268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102191007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[0]R flush_numbers[0..12]: [34, 26, 30, 28, 31, 33, 33, 34, 34, 28, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.10229605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[0]R generation[0..12]: [23, 17, 20, 18, 20, 22, 22, 23, 23, 18, 18, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102315739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102332328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[1]R flush_numbers[0..12]: [34, 26, 30, 28, 31, 33, 33, 34, 34, 28, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102347867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[1]R generation[0..12]: [23, 17, 20, 18, 20, 22, 22, 23, 23, 18, 18, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102363156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102379325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[2]R flush_numbers[0..12]: [34, 26, 30, 28, 31, 33, 33, 34, 34, 28, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102394553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[2]R generation[0..12]: [23, 17, 20, 18, 20, 22, 22, 23, 23, 18, 18, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102409962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102426191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:32:47.10243954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102458889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102472198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102485697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"":"downstairs","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102499666Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102512305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"0","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102528184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"1","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102542113ZDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"client":"2","":"downstairs","session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102560172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102573631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"54c1c4c3-37d7-4290-87e4-4a6ea87feda6 is now active with session: de78ced7-7c99-4f2d-8484-48406f612219","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.10258891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102601799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296,"session_id":"de78ced7-7c99-4f2d-8484-48406f612219"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102615148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} {"msg":"Activated sub_volume 54c1c4c3-37d7-4290-87e4-4a6ea87feda6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:47.102626778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 28672 data: 4 4 4 4 4 4 3 02/30 Write block 163 len 24576 data: 6 4 5 5 4 3 03/30 Read block 318 len 12288 04/30 Read block 413 len 8192 05/30 Flush 06/30 Write block 245 len 8192 data: 4 3 07/30 Flush 08/30 Write block 424 len 4096 data: 5 09/30 Read block 186 len 24576 10/30 Read block 331 len 8192 11/30 Write block 349 len 24576 data: 3 3 3 3 3 4 12/30 Write block 180 len 32768 data: 4 6 6 4 3 3 2 2 13/30 Write block 210 len 4096 data: 3 14/30 Read block 236 len 24576 15/30 Write block 38 len 40960 data: 5 5 5 3 3 3 3 3 3 3 16/30 Read block 145 len 12288 17/30 Write block 391 len 20480 data: 3 4 4 4 4 18/30 Write block 203 len 32768 data: 5 5 4 4 3 3 4 4 19/30 Write block 39 len 4096 data: 6 20/30 Read block 471 len 28672 21/30 Write block 487 len 12288 data: 3 3 4 22/30 Read block 501 len 28672 23/30 Read block 85 len 20480 24/30 Write block 171 len 4096 data: 3 25/30 Write block 96 len 20480 data: 4 4 4 3 3 26/30 Write block 204 len 32768 data: 6 5 5 4 4 5 5 3 27/30 Read block 182 len 20480 28/30 Read block 348 len 40960 29/30 Write block 164 len 40960 data: 5 6 6 5 4 4 3 4 4 4 30/30 Write block 12 len 36864 data: 3 3 3 3 3 5 5 5 5 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 4 28 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:47.11430604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 34 34 34 F T T <--- 1 020-039 17 17 17 26 26 26 F T T <--- 2 040-059 20 20 20 30 30 30 F T T <--- 3 060-079 18 18 18 28 28 28 F F F 4 080-099 20 20 20 31 31 31 F T T <--- 5 100-119 22 22 22 33 33 33 F T T <--- 6 120-139 22 22 22 33 33 33 F F F 7 140-159 23 23 23 34 34 34 F F F 8 160-179 23 24 24 34 35 35 T T T <--- 9 180-199 18 18 18 28 28 28 F T T <--- 10 200-219 18 18 18 28 28 28 F T T <--- 11 220-239 23 23 23 34 34 34 F F F 12 240-259 23 24 24 34 36 36 F F F <--- 13 260-279 22 22 22 33 33 33 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 23 23 23 34 34 34 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 20 20 20 30 30 30 F T T <--- 18 360-379 20 20 20 31 31 31 F F F 19 380-399 23 23 23 34 34 34 F T T <--- 20 400-419 18 18 18 28 28 28 F F F 21 420-439 16 24 24 24 35 35 T T T <--- 22 440-459 23 23 23 34 34 34 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 21 21 21 32 32 32 F T T <--- 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 23 23 23 34 34 34 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 24, Max flush: 36 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 37 37 37 F F F 1 020-039 25 25 25 37 37 37 F F F 2 040-059 25 25 25 37 37 37 F F F 3 060-079 18 18 18 28 28 28 F F F 4 080-099 25 25 25 37 37 37 F F F 5 100-119 25 25 25 37 37 37 F F F 6 120-139 22 22 22 33 33 33 F F F 7 140-159 23 23 23 34 34 34 F F F 8 160-179 25 25 25 37 37 37 F F F 9 180-199 25 25 25 37 37 37 F F F 10 200-219 25 25 25 37 37 37 F F F 11 220-239 23 23 23 34 34 34 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 22 22 22 33 33 33 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 23 23 23 34 34 34 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 25 25 25 37 37 37 F F F 18 360-379 20 20 20 31 31 31 F F F 19 380-399 25 25 25 37 37 37 F F F 20 400-419 18 18 18 28 28 28 F F F 21 420-439 25 25 25 37 37 37 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 25 25 25 37 37 37 F F F 25 500-519 18 18 18 28 28 28 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 F F F 28 560-579 23 23 23 34 34 34 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 25, Max flush: 37 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.464106728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476124597Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476435065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476472633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476485332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476503151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"Crucible b48e2296-3ad5-47f7-8b6f-85554a6259fb has session id: d0847403-a69a-45b5-bdac-95c69a62bf11","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476828529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Upstairs opts: Upstairs UUID: b48e2296-3ad5-47f7-8b6f-85554a6259fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476863886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Crucible stats registered with UUID: b48e2296-3ad5-47f7-8b6f-85554a6259fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.476882025Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:51.476895264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477077642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477113389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477128988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477214292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477232591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"b48e2296-3ad5-47f7-8b6f-85554a6259fb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.47725287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477266259Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477280428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477294357Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477308066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477322175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477336684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"ds_connection connected from Ok(127.0.0.1:42328)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477354903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"ds_connection connected from Ok(127.0.0.1:49560)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.477372831Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"ds_connection connected from Ok(127.0.0.1:37254)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.47738959Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478108631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478158638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478293129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478498455Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478537682Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[0]R flush_numbers[0..12]: [37, 37, 37, 28, 37, 37, 33, 34, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478699561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[0]R generation[0..12]: [25, 25, 25, 18, 25, 25, 22, 23, 25, 25, 25, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.47871656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478733268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[1]R flush_numbers[0..12]: [37, 37, 37, 28, 37, 37, 33, 34, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478748967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[1]R generation[0..12]: [25, 25, 25, 18, 25, 25, 22, 23, 25, 25, 25, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478764266Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478780485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[2]R flush_numbers[0..12]: [37, 37, 37, 28, 37, 37, 33, 34, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478795984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[2]R generation[0..12]: [25, 25, 25, 18, 25, 25, 22, 23, 25, 25, 25, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478814033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478830382Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:32:51.478844111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478864439Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478877549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478890758Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"":"downstairs","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478905117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478918006Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"0","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478932415Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"1","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478946584Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"client":"2","":"downstairs","session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478963183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478976542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"b48e2296-3ad5-47f7-8b6f-85554a6259fb is now active with session: d0847403-a69a-45b5-bdac-95c69a62bf11","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.478991191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.47900396Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304,"session_id":"d0847403-a69a-45b5-bdac-95c69a62bf11"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.479017359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} {"msg":"Activated sub_volume b48e2296-3ad5-47f7-8b6f-85554a6259fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:51.479028988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 28672 data: 2 2 3 3 3 4 5 02/30 Write block 263 len 20480 data: 4 4 5 3 2 03/30 Write block 133 len 36864 data: 2 3 3 3 3 4 3 3 3 04/30 Write block 26 len 20480 data: 2 3 3 4 4 05/30 Write block 393 len 32768 data: 5 5 5 4 3 3 3 3 06/30 Write block 355 len 32768 data: 5 5 4 5 5 6 6 6 07/30 Write block 184 len 28672 data: 4 4 3 3 2 2 2 08/30 Write block 127 len 40960 data: 4 3 3 3 2 2 3 4 4 4 09/30 Read block 344 len 24576 10/30 Read block 499 len 4096 11/30 Write block 199 len 16384 data: 7 6 4 5 12/30 Flush 13/30 Write block 410 len 40960 data: 4 5 4 4 5 5 6 5 4 6 14/30 Read block 399 len 40960 15/30 Write block 388 len 24576 data: 4 5 4 4 5 6 16/30 Flush 17/30 Write block 574 len 4096 data: 4 18/30 Read block 259 len 40960 19/30 Read block 43 len 4096 20/30 Write block 34 len 24576 data: 4 4 4 4 6 7 21/30 Flush 22/30 Flush 23/30 Write block 566 len 8192 data: 2 2 24/30 Flush 25/30 Write block 497 len 16384 data: 3 4 5 5 26/30 Write block 136 len 24576 data: 5 4 5 4 4 4 27/30 Read block 56 len 32768 28/30 Write block 545 len 24576 data: 3 2 2 2 2 2 29/30 Read block 37 len 36864 30/30 Write block 73 len 20480 data: 5 5 5 4 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 24 26 Done 29 29 6 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:51.490681552Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 37 37 37 F F F 1 020-039 26 26 25 40 40 37 F F T <--- 2 040-059 25 25 25 37 37 37 F F F 3 060-079 18 18 18 28 28 28 T T F <--- 4 080-099 25 25 25 37 37 37 F F F 5 100-119 25 25 25 37 37 37 F F F 6 120-139 26 26 22 38 38 33 T T T <--- 7 140-159 26 26 23 38 38 34 T T T <--- 8 160-179 26 26 25 38 38 37 F F T <--- 9 180-199 26 26 25 38 38 37 F F F <--- 10 200-219 26 26 25 38 38 37 F F F <--- 11 220-239 23 23 23 34 34 34 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 26 26 22 38 38 33 F F T <--- 14 280-299 20 20 20 30 30 30 F F F 15 300-319 23 23 23 34 34 34 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 25 38 38 37 F F T <--- 18 360-379 26 26 20 38 38 31 F F T <--- 19 380-399 26 26 25 39 39 37 F F T <--- 20 400-419 26 26 18 39 39 28 F F T <--- 21 420-439 25 25 25 37 37 37 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 25 25 25 37 37 37 T T F <--- 25 500-519 18 18 18 28 28 28 T T F <--- 26 520-539 18 18 18 28 28 28 F F F 27 540-559 16 16 16 23 23 23 T T F <--- 28 560-579 26 26 23 42 42 34 F F F <--- 29 580-599 22 22 22 33 33 33 F F F Max gen: 26, Max flush: 42 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 37 37 37 F F F 1 020-039 26 26 26 40 40 40 F F F 2 040-059 25 25 25 37 37 37 F F F 3 060-079 27 27 27 43 43 43 F F F 4 080-099 25 25 25 37 37 37 F F F 5 100-119 25 25 25 37 37 37 F F F 6 120-139 27 27 27 43 43 43 F F F 7 140-159 27 27 27 43 43 43 F F F 8 160-179 26 26 26 38 38 38 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 26 26 26 38 38 38 F F F 11 220-239 23 23 23 34 34 34 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 26 26 26 38 38 38 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 23 23 23 34 34 34 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 F F F 18 360-379 26 26 26 38 38 38 F F F 19 380-399 26 26 26 39 39 39 F F F 20 400-419 26 26 26 39 39 39 F F F 21 420-439 25 25 25 37 37 37 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 21 21 21 32 32 32 F F F 24 480-499 27 27 27 43 43 43 F F F 25 500-519 27 27 27 43 43 43 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 27 27 27 43 43 43 F F F 28 560-579 26 26 26 42 42 42 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 27, Max flush: 43 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.822061769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.833824755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834174991Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834213568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834226237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834243976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"Crucible beffb3ba-d416-46b9-a611-b01a43d149e9 has session id: 2eb11204-f028-47fd-afc8-5e262b117e2e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834566484Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Upstairs opts: Upstairs UUID: beffb3ba-d416-46b9-a611-b01a43d149e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834608161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Crucible stats registered with UUID: beffb3ba-d416-46b9-a611-b01a43d149e9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83462419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:55.834638329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834806588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834831866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834846705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834966447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.834984976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"beffb3ba-d416-46b9-a611-b01a43d149e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835005404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835018493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835033002Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835047331Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83506135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835075839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835102078Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49284)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835128836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64304)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835143875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35485)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835158924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835919152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83594844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.835965559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836335043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83638031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[0]R flush_numbers[0..12]: [37, 40, 37, 43, 37, 37, 43, 43, 38, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836450895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[0]R generation[0..12]: [25, 26, 25, 27, 25, 25, 27, 27, 26, 26, 26, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836470904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836487613Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[1]R flush_numbers[0..12]: [37, 40, 37, 43, 37, 37, 43, 43, 38, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836503132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[1]R generation[0..12]: [25, 26, 25, 27, 25, 25, 27, 27, 26, 26, 26, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836518381Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83653474Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[2]R flush_numbers[0..12]: [37, 40, 37, 43, 37, 37, 43, 43, 38, 38, 38, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836550269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[2]R generation[0..12]: [25, 26, 25, 27, 25, 25, 27, 27, 26, 26, 26, 23]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836565448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836581786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836595365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836609824Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":Wait for a query_work_queue command to finish before sending IO 1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836628673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836642212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"":"downstairs","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836656761Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.83666957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"0","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836685429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"1","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836699558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"client":"2","":"downstairs","session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836713747Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "crucible","level":30,"time":"2026-03-16T21:32:55.836730256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"beffb3ba-d416-46b9-a611-b01a43d149e9 is now active with session: 2eb11204-f028-47fd-afc8-5e262b117e2e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836745535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836758274Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312,"session_id":"2eb11204-f028-47fd-afc8-5e262b117e2e"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836771583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} {"msg":"Activated sub_volume beffb3ba-d416-46b9-a611-b01a43d149e9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:32:55.836783253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 384 len 8192 data: 3 3 02/30 Flush 03/30 Write block 574 len 8192 data: 5 4 04/30 Write block 124 len 24576 data: 2 2 3 5 4 4 05/30 Write block 466 len 8192 data: 4 4 06/30 Write block 540 len 12288 data: 3 3 3 07/30 Read block 371 len 32768 08/30 Read block 392 len 24576 09/30 Read block 82 len 40960 10/30 Write block 218 len 12288 data: 4 4 4 11/30 Write block 584 len 12288 data: 3 3 2 12/30 Read block 561 len 20480 13/30 Read block 50 len 32768 14/30 Read block 169 len 24576 15/30 Flush 16/30 Write block 17 len 32768 data: 6 6 6 6 4 4 5 5 17/30 Write block 304 len 32768 data: 5 5 5 5 5 5 4 4 18/30 Write block 131 len 8192 data: 3 3 19/30 Read block 106 len 24576 20/30 Read block 48 len 4096 21/30 Write block 241 len 40960 data: 3 5 5 5 5 4 3 3 3 3 22/30 Flush 23/30 Write block 481 len 36864 data: 4 5 5 2 2 2 4 4 5 24/30 Flush 25/30 Write block 227 len 20480 data: 5 5 5 5 5 26/30 Write block 106 len 36864 data: 2 2 2 2 2 3 3 3 3 27/30 Read block 161 len 16384 28/30 Write block 172 len 16384 data: 5 5 4 5 29/30 Read block 109 len 8192 30/30 Read block 160 len 20480 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:32:55.84867697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 25 46 46 37 F F F <--- 1 020-039 28 28 26 46 46 40 F F F <--- 2 040-059 25 25 25 37 37 37 F F F 3 060-079 27 27 27 43 43 43 F F F 4 080-099 25 25 25 37 37 37 F F F 5 100-119 25 25 25 37 37 37 T T F <--- 6 120-139 28 28 27 46 46 43 F F F <--- 7 140-159 27 27 27 43 43 43 F F F 8 160-179 26 26 26 38 38 38 T T F <--- 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 26 45 45 38 F F F <--- 11 220-239 28 28 23 45 45 34 T T F <--- 12 240-259 28 28 24 46 46 36 F F F <--- 13 260-279 26 26 26 38 38 38 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 28 28 23 46 46 34 F F F <--- 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 F F F 18 360-379 26 26 26 38 38 38 F F F 19 380-399 28 28 26 44 44 39 F F F <--- 20 400-419 26 26 26 39 39 39 F F F 21 420-439 25 25 25 37 37 37 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 28 28 21 45 45 32 F F F <--- 24 480-499 28 28 27 47 47 43 F F F <--- 25 500-519 27 27 27 43 43 43 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 28 28 27 45 45 43 F F F <--- 28 560-579 28 28 26 45 45 42 F F F <--- 29 580-599 28 28 22 45 45 33 F F F <--- Max gen: 28, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 46 46 46 F F F 1 020-039 28 28 28 46 46 46 F F F 2 040-059 25 25 25 37 37 37 F F F 3 060-079 27 27 27 43 43 43 F F F 4 080-099 25 25 25 37 37 37 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 28 28 28 46 46 46 F F F 7 140-159 27 27 27 43 43 43 F F F 8 160-179 29 29 29 48 48 48 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 29 29 29 48 48 48 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 26 26 26 38 38 38 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 28 28 28 46 46 46 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 F F F 18 360-379 26 26 26 38 38 38 F F F 19 380-399 28 28 28 44 44 44 F F F 20 400-419 26 26 26 39 39 39 F F F 21 420-439 25 25 25 37 37 37 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 28 28 28 45 45 45 F F F 24 480-499 28 28 28 47 47 47 F F F 25 500-519 27 27 27 43 43 43 F F F 26 520-539 18 18 18 28 28 28 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.171414887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183062851Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.1833791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183416777Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183429406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183447255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"Crucible 21e9d144-a0e4-48d3-ad71-372c1eafea88 has session id: 9f6affb5-6a2d-43dd-8c80-82926d48d627","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183826389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Upstairs opts: Upstairs UUID: 21e9d144-a0e4-48d3-ad71-372c1eafea88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183858567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Crucible stats registered with UUID: 21e9d144-a0e4-48d3-ad71-372c1eafea88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.183875676Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:00.183888905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184057164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184093081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.18410903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184203154Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184221262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"21e9d144-a0e4-48d3-ad71-372c1eafea88 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184238281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.18425119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184265329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184279438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184293217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184307646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184321595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"ds_connection connected from Ok(127.0.0.1:39076)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184367792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"ds_connection connected from Ok(127.0.0.1:39447)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.18439854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"ds_connection connected from Ok(127.0.0.1:48890)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.184414979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185019888Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185066525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185204665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185374044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185414541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[0]R flush_numbers[0..12]: [46, 46, 37, 43, 37, 48, 46, 43, 48, 38, 45, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.18557704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[0]R generation[0..12]: [28, 28, 25, 27, 25, 29, 28, 27, 29, 26, 28, 29]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185594748Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185611207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[1]R flush_numbers[0..12]: [46, 46, 37, 43, 37, 48, 46, 43, 48, 38, 45, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185626896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[1]R generation[0..12]: [28, 28, 25, 27, 25, 29, 28, 27, 29, 26, 28, 29]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185642385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185658894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[2]R flush_numbers[0..12]: [46, 46, 37, 43, 37, 48, 46, 43, 48, 38, 45, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185674363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[2]R generation[0..12]: [28, 28, 25, 27, 25, 29, 28, 27, 29, 26, 28, 29]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185692092Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185708471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185727579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185741738Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185755148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185768367Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"":"downstairs","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185782506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185795335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"0","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185809334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"1","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185826153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"client":"2","":"downstairs","session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185840222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185853791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"21e9d144-a0e4-48d3-ad71-372c1eafea88 is now active with session: 9f6affb5-6a2d-43dd-8c80-82926d48d627","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.18586855Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185880999Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320,"session_id":"9f6affb5-6a2d-43dd-8c80-82926d48d627"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185894318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} {"msg":"Activated sub_volume 21e9d144-a0e4-48d3-ad71-372c1eafea88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:00.185906057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 28672 data: 3 3 2 3 3 4 4 02/30 Flush 03/30 Write block 112 len 20480 data: 4 4 4 3 3 04/30 Write block 517 len 8192 data: 4 4 05/30 Read block 340 len 32768 06/30 Read block 12 len 12288 07/30 Read block 483 len 20480 08/30 Read block 308 len 16384 09/30 Write block 3 len 40960 data: 3 3 3 3 3 3 3 3 3 4 10/30 Read block 502 len 40960 11/30 Write block 486 len 40960 data: 3 5 5 6 4 4 4 4 4 4 12/30 Read block 65 len 20480 13/30 Read block 15 len 40960 14/30 Read block 268 len 40960 15/30 Flush 16/30 Write block 396 len 36864 data: 5 4 4 4 4 3 5 5 4 17/30 Read block 172 len 32768 18/30 Write block 511 len 40960 data: 3 2 3 3 4 4 5 5 3 3 19/30 Write block 1 len 20480 data: 3 3 4 4 4 20/30 Read block 403 len 32768 21/30 Read block 332 len 32768 22/30 Read block 469 len 4096 23/30 Write block 386 len 32768 data: 3 4 5 6 5 5 6 7 24/30 Write block 36 len 24576 data: 5 5 7 8 6 4 25/30 Write block 420 len 20480 data: 6 5 4 5 6 26/30 Read block 320 len 20480 27/30 Write block 234 len 16384 data: 4 4 3 2 28/30 Write block 149 len 32768 data: 3 3 3 4 4 3 4 4 29/30 Write block 96 len 28672 data: 5 5 5 4 4 3 3 30/30 Write block 404 len 20480 data: 5 4 4 5 6 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:00.198944166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 28 50 50 46 T T F <--- 1 020-039 28 28 28 46 46 46 T T F <--- 2 040-059 25 25 25 37 37 37 T T F <--- 3 060-079 27 27 27 43 43 43 F F F 4 080-099 25 25 25 37 37 37 T T F <--- 5 100-119 30 30 29 50 50 48 T T F <--- 6 120-139 28 28 28 46 46 46 F F F 7 140-159 30 30 27 49 49 43 T T F <--- 8 160-179 29 29 29 48 48 48 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 29 29 29 48 48 48 T T F <--- 12 240-259 28 28 28 46 46 46 F F F 13 260-279 26 26 26 38 38 38 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 28 28 28 46 46 46 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 F F F 18 360-379 26 26 26 38 38 38 F F F 19 380-399 28 28 28 44 44 44 T T F <--- 20 400-419 26 26 26 39 39 39 T T F <--- 21 420-439 25 25 25 37 37 37 T T F <--- 22 440-459 23 23 23 34 34 34 F F F 23 460-479 28 28 28 45 45 45 F F F 24 480-499 30 30 28 50 50 47 F F F <--- 25 500-519 30 30 27 50 50 43 T T F <--- 26 520-539 18 18 18 28 28 28 T T F <--- 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 30, Max flush: 50 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 51 51 51 F F F 1 020-039 31 31 31 51 51 51 F F F 2 040-059 31 31 31 51 51 51 F F F 3 060-079 27 27 27 43 43 43 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 31 31 31 51 51 51 F F F 6 120-139 28 28 28 46 46 46 F F F 7 140-159 31 31 31 51 51 51 F F F 8 160-179 29 29 29 48 48 48 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 26 26 26 38 38 38 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 28 28 28 46 46 46 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 F F F 18 360-379 26 26 26 38 38 38 F F F 19 380-399 31 31 31 51 51 51 F F F 20 400-419 31 31 31 51 51 51 F F F 21 420-439 31 31 31 51 51 51 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 28 28 28 45 45 45 F F F 24 480-499 30 30 30 50 50 50 F F F 25 500-519 31 31 31 51 51 51 F F F 26 520-539 31 31 31 51 51 51 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 31, Max flush: 51 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.524399608Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536102138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536428786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536468633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536480673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536498621Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"Crucible a69d4c42-931f-4148-b5e3-83ac63b668d7 has session id: 466610fa-8854-479d-a04e-756894d3697f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536843158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Upstairs opts: Upstairs UUID: a69d4c42-931f-4148-b5e3-83ac63b668d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536878405Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Crucible stats registered with UUID: a69d4c42-931f-4148-b5e3-83ac63b668d7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.536894014Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:04.536907393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537075722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.53711143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537127098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.53724953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537266899Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"a69d4c42-931f-4148-b5e3-83ac63b668d7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537283518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537296647Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537310476Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537324655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537338294Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537383351Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537406349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45318)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537421978Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55951)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537437017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33681)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.537451466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538198245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538226873Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538244102Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538555641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538597328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[0]R flush_numbers[0..12]: [51, 51, 51, 43, 51, 51, 46, 51, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538654634Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[0]R generation[0..12]: [31, 31, 31, 27, 31, 31, 28, 31, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538671473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538687932Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[1]R flush_numbers[0..12]: [51, 51, 51, 43, 51, 51, 46, 51, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538703421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[1]R generation[0..12]: [31, 31, 31, 27, 31, 31, 28, 31, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.53871863Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538734759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[2]R flush_numbers[0..12]: [51, 51, 51, 43, 51, 51, 46, 51, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538749768Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[2]R generation[0..12]: [31, 31, 31, 27, 31, 31, 28, 31, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538764877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538780785Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538794195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538808284Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538827042Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538840571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"":"downstairs","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.53885472Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.53886742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"0","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538883258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"1","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538897407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"client":"2","":"downstairs","session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538911917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Set Downstairs and Upstairs active after reconciliation"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538928595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"a69d4c42-931f-4148-b5e3-83ac63b668d7 is now active with session: 466610fa-8854-479d-a04e-756894d3697f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538943954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538956743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328,"session_id":"466610fa-8854-479d-a04e-756894d3697f"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538970123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} {"msg":"Activated sub_volume a69d4c42-931f-4148-b5e3-83ac63b668d7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:04.538994201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 32768 data: 5 4 4 4 4 3 3 3 02/30 Read block 293 len 24576 03/30 Read block 389 len 36864 04/30 Write block 426 len 40960 data: 5 5 5 4 3 3 3 2 3 3 05/30 Read block 78 len 40960 06/30 Read block 485 len 20480 07/30 Read block 479 len 28672 08/30 Read block 135 len 8192 09/30 Write block 136 len 36864 data: 6 5 6 5 5 5 3 2 4 10/30 Write block 366 len 12288 data: 5 6 5 11/30 Read block 540 len 36864 12/30 Write block 478 len 20480 data: 4 4 4 5 6 13/30 Write block 117 len 16384 data: 3 3 2 2 14/30 Write block 510 len 12288 data: 4 4 3 15/30 Read block 200 len 16384 16/30 Write block 510 len 36864 data: 5 5 4 4 4 5 5 6 6 17/30 Write block 64 len 20480 data: 4 4 3 3 4 18/30 Write block 413 len 4096 data: 5 19/30 Write block 246 len 36864 data: 5 4 4 4 4 2 3 3 3 20/30 Write block 392 len 28672 data: 7 8 6 6 6 5 5 21/30 Read block 146 len 8192 22/30 Read block 461 len 16384 23/30 Flush 24/30 Write block 356 len 8192 data: 6 5 25/30 Read block 234 len 28672 26/30 Read block 124 len 4096 27/30 Write block 425 len 12288 data: 5 6 6 28/30 Read block 16 len 12288 29/30 Read block 176 len 24576 30/30 Write block 312 len 12288 data: 3 2 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:04.551313879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 51 51 51 F F F 1 020-039 31 31 31 51 51 51 F F F 2 040-059 31 31 31 51 51 51 F F F 3 060-079 32 27 32 52 43 52 F F F <--- 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 31 32 52 51 52 F F F <--- 6 120-139 32 28 32 52 46 52 F F F <--- 7 140-159 32 31 32 52 51 52 F F F <--- 8 160-179 29 29 29 48 48 48 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 32 28 32 52 46 52 F F F <--- 13 260-279 26 26 26 38 38 38 F F F 14 280-299 32 20 32 52 30 52 F T F <--- 15 300-319 28 28 28 46 46 46 T F T <--- 16 320-339 22 22 22 33 33 33 F F F 17 340-359 26 26 26 38 38 38 T F T <--- 18 360-379 32 26 32 52 38 52 F F F <--- 19 380-399 32 31 32 52 51 52 F F F <--- 20 400-419 32 31 32 52 51 52 F F F <--- 21 420-439 32 31 32 52 51 52 T F T <--- 22 440-459 23 23 23 34 34 34 F F F 23 460-479 32 28 32 52 45 52 F F F <--- 24 480-499 32 30 32 52 50 52 F F F <--- 25 500-519 32 31 32 52 51 52 F F F <--- 26 520-539 31 31 31 51 51 51 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 32, Max flush: 52 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 51 51 51 F F F 1 020-039 31 31 31 51 51 51 F F F 2 040-059 31 31 31 51 51 51 F F F 3 060-079 32 32 32 52 52 52 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 52 52 52 F F F 6 120-139 32 32 32 52 52 52 F F F 7 140-159 32 32 32 52 52 52 F F F 8 160-179 29 29 29 48 48 48 F F F 9 180-199 26 26 26 38 38 38 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 26 26 26 38 38 38 F F F 14 280-299 32 32 32 52 52 52 F F F 15 300-319 33 33 33 53 53 53 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 33 33 33 53 53 53 F F F 18 360-379 32 32 32 52 52 52 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 33 33 33 53 53 53 F F F 22 440-459 23 23 23 34 34 34 F F F 23 460-479 32 32 32 52 52 52 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 32 32 32 52 52 52 F F F 26 520-539 31 31 31 51 51 51 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 33, Max flush: 53 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.873687561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.885439988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.885772196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.885815373Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.885828282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.88584658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"Crucible 031c052a-69b5-4a34-b810-310844ec2520 has session id: a1ca658e-bb90-4d39-a28b-c9a8fef7b77d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886176148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Upstairs opts: Upstairs UUID: 031c052a-69b5-4a34-b810-310844ec2520, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886236744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Crucible stats registered with UUID: 031c052a-69b5-4a34-b810-310844ec2520","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886253443Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:08.886266872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.88644545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886482267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886497916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886602829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886621088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"031c052a-69b5-4a34-b810-310844ec2520 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886654455Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886668624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886682773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886696822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886710661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.88672503Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886739099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48533)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886786776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37462)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886805395Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50696)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.886820294Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887561813Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887590771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.88760785Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887918359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887942617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[0]R flush_numbers[0..12]: [51, 51, 51, 52, 51, 52, 52, 52, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887963196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[0]R generation[0..12]: [31, 31, 31, 32, 31, 32, 32, 32, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.887991044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888008413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[1]R flush_numbers[0..12]: [51, 51, 51, 52, 51, 52, 52, 52, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888024302Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[1]R generation[0..12]: [31, 31, 31, 32, 31, 32, 32, 32, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888039751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888056409Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[2]R flush_numbers[0..12]: [51, 51, 51, 52, 51, 52, 52, 52, 48, 38, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888071708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[2]R generation[0..12]: [31, 31, 31, 32, 31, 32, 32, 32, 29, 26, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888089657Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888106326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888120115Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:33:08.888134374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888152803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888166372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"":"downstairs","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888180731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.88819361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"0","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888213909Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"1","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888228038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"client":"2","":"downstairs","session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888242397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888256096Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"031c052a-69b5-4a34-b810-310844ec2520 is now active with session: a1ca658e-bb90-4d39-a28b-c9a8fef7b77d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888274045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888286974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336,"session_id":"a1ca658e-bb90-4d39-a28b-c9a8fef7b77d"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888300493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} {"msg":"Activated sub_volume 031c052a-69b5-4a34-b810-310844ec2520","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:08.888312392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 30 len 20480 data: 5 4 5 4 5 02/30 Write block 271 len 16384 data: 4 4 4 4 03/30 Flush 04/30 Write block 256 len 4096 data: 2 05/30 Read block 262 len 20480 06/30 Read block 560 len 28672 07/30 Read block 152 len 8192 08/30 Write block 175 len 36864 data: 6 5 5 5 4 5 7 7 5 09/30 Read block 52 len 4096 10/30 Flush 11/30 Write block 357 len 32768 data: 6 6 6 7 7 7 5 5 12/30 Read block 378 len 36864 13/30 Read block 126 len 16384 14/30 Read block 353 len 28672 15/30 Flush 16/30 Write block 47 len 24576 data: 4 4 5 5 4 5 17/30 Write block 293 len 16384 data: 4 4 4 3 18/30 Read block 142 len 24576 19/30 Flush 20/30 Write block 12 len 8192 data: 5 4 21/30 Write block 366 len 24576 data: 6 7 6 5 5 5 22/30 Read block 3 len 32768 23/30 Flush 24/30 Write block 575 len 12288 data: 5 4 3 25/30 Read block 332 len 28672 26/30 Write block 60 len 12288 data: 3 3 3 27/30 Write block 448 len 40960 data: 4 3 4 4 5 5 5 4 4 4 28/30 Write block 429 len 40960 data: 5 4 4 4 3 4 4 3 4 4 29/30 Read block 547 len 36864 30/30 Read block 429 len 40960 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:08.900374198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 31 34 58 51 58 F F F <--- 1 020-039 34 31 34 54 51 54 F F F <--- 2 040-059 34 31 34 57 51 57 F F F <--- 3 060-079 32 32 32 52 52 52 T F T <--- 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 52 52 52 F F F 6 120-139 32 32 32 52 52 52 F F F 7 140-159 32 32 32 52 52 52 F F F 8 160-179 34 29 34 55 48 55 F F F <--- 9 180-199 34 26 34 55 38 55 F F F <--- 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 34 32 34 55 52 55 F F F <--- 13 260-279 34 26 34 54 38 54 F F F <--- 14 280-299 34 32 34 57 52 57 F F F <--- 15 300-319 33 33 33 53 53 53 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 34 33 34 56 53 56 F F F <--- 18 360-379 34 32 34 58 52 58 F F F <--- 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 33 33 33 53 53 53 T F T <--- 22 440-459 23 23 23 34 34 34 T F T <--- 23 460-479 32 32 32 52 52 52 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 32 32 32 52 52 52 F F F 26 520-539 31 31 31 51 51 51 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 T F T <--- 29 580-599 28 28 28 45 45 45 F F F Max gen: 34, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 34 34 34 57 57 57 F F F 3 060-079 35 35 35 59 59 59 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 52 52 52 F F F 6 120-139 32 32 32 52 52 52 F F F 7 140-159 32 32 32 52 52 52 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 34 34 34 55 55 55 F F F 13 260-279 34 34 34 54 54 54 F F F 14 280-299 34 34 34 57 57 57 F F F 15 300-319 33 33 33 53 53 53 F F F 16 320-339 22 22 22 33 33 33 F F F 17 340-359 34 34 34 56 56 56 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 35 35 35 59 59 59 F F F 22 440-459 35 35 35 59 59 59 F F F 23 460-479 32 32 32 52 52 52 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 32 32 32 52 52 52 F F F 26 520-539 31 31 31 51 51 51 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 35 35 35 59 59 59 F F F 29 580-599 28 28 28 45 45 45 F F F Max gen: 35, Max flush: 59 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.225113348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:13.235889222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.235943598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.246511006Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.246815595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.246853923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.246865582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.246883321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"Crucible c9a773c3-d1bf-4b4d-8b47-10fbe7640f03 has session id: 6e63d7d4-f34a-47d3-a9af-aa52a1639bf9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247229337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Upstairs opts: Upstairs UUID: c9a773c3-d1bf-4b4d-8b47-10fbe7640f03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247283483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Crucible stats registered with UUID: c9a773c3-d1bf-4b4d-8b47-10fbe7640f03","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247299462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:13.247312421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247489019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247515117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247529866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247658148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247675766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"c9a773c3-d1bf-4b4d-8b47-10fbe7640f03 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247690365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247703385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247717194Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247731243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247744532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247758651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.24777292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61275)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247809697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:65131)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247826046Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57921)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.247841105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.248639171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.248680878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.248697557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249000696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249043673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[0]R flush_numbers[0..12]: [58, 54, 57, 59, 51, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249107019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[0]R generation[0..12]: [34, 34, 34, 35, 31, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249123538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249140166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[1]R flush_numbers[0..12]: [58, 54, 57, 59, 51, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249155425Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[1]R generation[0..12]: [34, 34, 34, 35, 31, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249170454Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249186403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[2]R flush_numbers[0..12]: [58, 54, 57, 59, 51, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249201652Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[2]R generation[0..12]: [34, 34, 34, 35, 31, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249217031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.24923297Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249246279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249265958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249279397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249292716Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"":"downstairs","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249308505Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249321104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"0","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249335403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"1","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249352032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"client":"2","":"downstairs","session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249366821Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.24938012Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"c9a773c3-d1bf-4b4d-8b47-10fbe7640f03 is now active with session: 6e63d7d4-f34a-47d3-a9af-aa52a1639bf9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249395129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249407888Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344,"session_id":"6e63d7d4-f34a-47d3-a9af-aa52a1639bf9"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249421247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} {"msg":"Activated sub_volume c9a773c3-d1bf-4b4d-8b47-10fbe7640f03","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:13.249432506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 452 len 8192 data: 6 6 02/30 Write block 313 len 12288 data: 3 4 3 03/30 Read block 458 len 28672 04/30 Read block 379 len 40960 05/30 Read block 343 len 12288 06/30 Flush 07/30 Write block 53 len 8192 data: 3 2 08/30 Read block 349 len 12288 09/30 Write block 95 len 4096 data: 3 10/30 Write block 277 len 40960 data: 4 4 4 4 4 4 2 4 4 4 11/30 Write block 366 len 8192 data: 7 8 12/30 Read block 286 len 24576 13/30 Read block 315 len 8192 14/30 Write block 571 len 8192 data: 2 3 15/30 Read block 487 len 32768 16/30 Write block 503 len 40960 data: 7 6 5 5 6 5 5 6 6 5 17/30 Read block 367 len 32768 18/30 Flush 19/30 Write block 405 len 12288 data: 5 5 6 20/30 Write block 324 len 20480 data: 5 4 3 3 3 21/30 Read block 588 len 28672 22/30 Read block 226 len 20480 23/30 Read block 538 len 24576 24/30 Write block 253 len 8192 data: 4 4 25/30 Write block 436 len 8192 data: 4 5 26/30 Read block 258 len 12288 27/30 Read block 377 len 28672 28/30 Write block 527 len 32768 data: 3 3 3 4 3 3 3 3 29/30 Write block 498 len 24576 data: 5 6 6 5 5 8 30/30 Write block 581 len 16384 data: 2 2 3 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 8 Sent Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 2 34 Done 1 27 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:13.260339551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 34 36 36 57 61 61 F F F <--- 3 060-079 35 35 35 59 59 59 F F F 4 080-099 31 36 36 51 61 61 F F F <--- 5 100-119 32 32 32 52 52 52 F F F 6 120-139 32 32 32 52 52 52 F F F 7 140-159 32 32 32 52 52 52 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 34 34 34 55 55 55 F T T <--- 13 260-279 34 36 36 54 61 61 F F F <--- 14 280-299 34 36 36 57 61 61 F F F <--- 15 300-319 33 36 36 53 60 60 F F F <--- 16 320-339 22 22 22 33 33 33 F T T <--- 17 340-359 34 34 34 56 56 56 F F F 18 360-379 34 36 36 58 61 61 F F F <--- 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F T T <--- 21 420-439 35 35 35 59 59 59 F T T <--- 22 440-459 35 36 36 59 60 60 T F F <--- 23 460-479 32 32 32 52 52 52 F F F 24 480-499 32 32 32 52 52 52 F T T <--- 25 500-519 32 36 36 52 61 61 F T T <--- 26 520-539 31 31 31 51 51 51 F T T <--- 27 540-559 28 28 28 45 45 45 F F F 28 560-579 35 36 36 59 61 61 F F F <--- 29 580-599 28 28 28 45 45 45 F T T <--- Max gen: 36, Max flush: 61 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 36 36 36 61 61 61 F F F 3 060-079 35 35 35 59 59 59 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 32 32 32 52 52 52 F F F 6 120-139 32 32 32 52 52 52 F F F 7 140-159 32 32 32 52 52 52 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 36 36 36 61 61 61 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 34 34 34 56 56 56 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 37 37 37 62 62 62 F F F 21 420-439 37 37 37 62 62 62 F F F 22 440-459 36 36 36 60 60 60 F F F 23 460-479 32 32 32 52 52 52 F F F 24 480-499 37 37 37 62 62 62 F F F 25 500-519 37 37 37 62 62 62 F F F 26 520-539 37 37 37 62 62 62 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.599731321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.611698833Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612023401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612061568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612073537Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612091366Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"Crucible 5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98 has session id: 7bfe4182-7a69-434a-836f-44fa1aa6f365","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612434203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Upstairs opts: Upstairs UUID: 5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.6124681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Crucible stats registered with UUID: 5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612483669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:17.612497558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612664507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612690795Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612705224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612808737Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612829086Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612848324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612861024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612874863Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612888812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"2","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612902561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.61291672Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612930939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"ds_connection connected from Ok(127.0.0.1:47664)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612948598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"ds_connection connected from Ok(127.0.0.1:39748)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612963907Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"ds_connection connected from Ok(127.0.0.1:39271)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.612978786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.613782651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.613813538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"2","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.613830607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614174704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614189083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[0]R flush_numbers[0..12]: [58, 54, 61, 59, 61, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614222551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[0]R generation[0..12]: [34, 34, 36, 35, 36, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614239039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614255798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[1]R flush_numbers[0..12]: [58, 54, 61, 59, 61, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614271057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[1]R generation[0..12]: [34, 34, 36, 35, 36, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614286636Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614302935Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[2]R flush_numbers[0..12]: [58, 54, 61, 59, 61, 52, 52, 52, 55, 55, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614320734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[2]R generation[0..12]: [34, 34, 36, 35, 36, 32, 32, 32, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614336263Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614352402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"Wait for a query_work_queue command to finish before sending IO msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.61437103Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614385549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614398798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614411998Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"":"downstairs","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614425907Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614438446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"0","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614452125Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"1","":"downstairs","session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614465954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"client":"2","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614483003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614496302Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98 is now active with session: 7bfe4182-7a69-434a-836f-44fa1aa6f365","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614511011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.61452393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352,"session_id":"7bfe4182-7a69-434a-836f-44fa1aa6f365"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614538779Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} {"msg":"Activated sub_volume 5e7e007d-ff08-4dd9-88b5-8e9ef5e22e98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:17.614550138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 36864 data: 5 6 7 6 3 3 4 6 6 02/30 Read block 502 len 16384 03/30 Write block 184 len 16384 data: 5 5 4 4 04/30 Write block 189 len 40960 data: 3 3 2 4 4 4 4 5 6 7 05/30 Write block 113 len 36864 data: 5 5 4 4 4 4 3 3 2 06/30 Write block 136 len 20480 data: 7 6 7 6 6 07/30 Write block 77 len 20480 data: 4 2 2 2 3 08/30 Write block 10 len 8192 data: 4 4 09/30 Write block 88 len 24576 data: 2 2 3 3 2 3 10/30 Write block 428 len 16384 data: 6 6 5 5 11/30 Read block 176 len 8192 12/30 Write block 346 len 36864 data: 3 3 3 4 4 4 4 4 5 13/30 Write block 116 len 20480 data: 5 5 5 4 4 14/30 Write block 426 len 40960 data: 7 7 7 7 6 6 5 4 5 5 15/30 Read block 386 len 12288 16/30 Read block 72 len 32768 17/30 Read block 400 len 16384 18/30 Read block 103 len 8192 19/30 Write block 409 len 20480 data: 5 5 6 5 6 20/30 Read block 505 len 40960 21/30 Read block 305 len 24576 22/30 Read block 219 len 20480 23/30 Read block 582 len 12288 24/30 Write block 418 len 8192 data: 5 7 25/30 Read block 31 len 36864 26/30 Write block 56 len 12288 data: 2 3 3 27/30 Read block 9 len 12288 28/30 Write block 295 len 40960 data: 5 4 2 2 2 2 3 4 4 6 29/30 Read block 125 len 12288 30/30 Read block 182 len 32768 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:17.626413908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 T F T <--- 1 020-039 34 34 34 54 54 54 F F F 2 040-059 36 36 36 61 61 61 T F T <--- 3 060-079 35 35 35 59 59 59 T F T <--- 4 080-099 36 36 36 61 61 61 T F T <--- 5 100-119 32 32 32 52 52 52 T F T <--- 6 120-139 32 32 32 52 52 52 T F T <--- 7 140-159 32 32 32 52 52 52 T F T <--- 8 160-179 34 34 34 55 55 55 F F F 9 180-199 34 34 34 55 55 55 T T T 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 36 36 36 61 61 61 T F T <--- 15 300-319 36 36 36 60 60 60 T F T <--- 16 320-339 37 37 37 62 62 62 F F F 17 340-359 34 34 34 56 56 56 T F T <--- 18 360-379 36 36 36 61 61 61 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 37 37 37 62 62 62 T F T <--- 21 420-439 37 37 37 62 62 62 T F T <--- 22 440-459 36 36 36 60 60 60 F F F 23 460-479 32 32 32 52 52 52 F F F 24 480-499 37 37 37 62 62 62 T T T 25 500-519 37 37 37 62 62 62 F F F 26 520-539 37 37 37 62 62 62 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 37, Max flush: 62 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 38 38 38 63 63 63 F F F 4 080-099 38 38 38 63 63 63 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 38 38 38 63 63 63 F F F 7 140-159 38 38 38 63 63 63 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 38 38 38 63 63 63 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 51 51 51 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 38 38 38 63 63 63 F F F 15 300-319 38 38 38 63 63 63 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 38 38 38 63 63 63 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 38 38 38 63 63 63 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 60 60 60 F F F 23 460-479 32 32 32 52 52 52 F F F 24 480-499 38 38 38 63 63 63 F F F 25 500-519 37 37 37 62 62 62 F F F 26 520-539 37 37 37 62 62 62 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 38, Max flush: 63 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.956890786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.968602105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.968920284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.968958651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.9689711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.968989129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"Crucible 82c6acc1-95d6-4920-b0dc-befe6f81de3c has session id: 9e9099b9-722b-44d6-8b9b-931dddc5d41a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969342795Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Upstairs opts: Upstairs UUID: 82c6acc1-95d6-4920-b0dc-befe6f81de3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969378522Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Crucible stats registered with UUID: 82c6acc1-95d6-4920-b0dc-befe6f81de3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969393911Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:21.96940737Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969598307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969634785Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969650244Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969734968Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969752707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"82c6acc1-95d6-4920-b0dc-befe6f81de3c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969776325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969789004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969803053Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"1","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969816742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969830672Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.96984559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.9698599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39615)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969876628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41933)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969891827Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39822)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.969906896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.970636966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.970663565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.970707562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"1","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971006761Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971049898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[0]R flush_numbers[0..12]: [63, 54, 63, 63, 63, 63, 63, 63, 55, 63, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971115054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[0]R generation[0..12]: [38, 34, 38, 38, 38, 38, 38, 38, 34, 38, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971131743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971148501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[1]R flush_numbers[0..12]: [63, 54, 63, 63, 63, 63, 63, 63, 55, 63, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.97116416Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[1]R generation[0..12]: [38, 34, 38, 38, 38, 38, 38, 38, 34, 38, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971185229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971201658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[2]R flush_numbers[0..12]: [63, 54, 63, 63, 63, 63, 63, 63, 55, 63, 45, 51]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971216737Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[2]R generation[0..12]: [38, 34, 38, 38, 38, 38, 38, 38, 34, 38, 28, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971231866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971248045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Max found gen is 39","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-16T21:33:21.971261524Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Generation requested: 40 >= found:39","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971281072Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971294301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971307511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"":"downstairs","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971323649Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971336429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"0","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971350308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971366427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"client":"2","":"downstairs","session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971380556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971394035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"82c6acc1-95d6-4920-b0dc-befe6f81de3c is now active with session: 9e9099b9-722b-44d6-8b9b-931dddc5d41a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971408804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971421553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360,"session_id":"9e9099b9-722b-44d6-8b9b-931dddc5d41a"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971434732Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} {"msg":"Activated sub_volume 82c6acc1-95d6-4920-b0dc-befe6f81de3c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:21.971446131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 28672 data: 3 3 3 3 4 5 5 02/30 Write block 534 len 40960 data: 4 2 2 2 2 2 4 4 4 4 03/30 Read block 339 len 40960 04/30 Read block 296 len 24576 05/30 Write block 406 len 28672 data: 6 7 7 6 6 7 6 06/30 Read block 236 len 16384 07/30 Write block 206 len 24576 data: 6 5 5 6 6 4 08/30 Flush 09/30 Write block 478 len 20480 data: 5 5 6 7 8 10/30 Write block 143 len 40960 data: 3 5 4 2 2 3 4 4 4 5 11/30 Flush 12/30 Write block 342 len 28672 data: 2 2 2 2 4 4 4 13/30 Read block 589 len 24576 14/30 Read block 37 len 16384 15/30 Write block 375 len 28672 data: 4 4 5 5 4 5 4 16/30 Write block 172 len 24576 data: 6 6 5 7 6 6 17/30 Write block 203 len 32768 data: 6 7 6 7 6 6 7 7 18/30 Read block 527 len 32768 19/30 Read block 472 len 8192 20/30 Flush 21/30 Write block 191 len 20480 data: 3 5 5 5 5 22/30 Read block 104 len 32768 23/30 Flush 24/30 Write block 228 len 16384 data: 6 6 6 6 25/30 Write block 220 len 36864 data: 5 4 5 5 5 5 5 6 7 26/30 Flush 27/30 Write block 480 len 24576 data: 7 8 9 7 4 4 28/30 Write block 488 len 28672 data: 7 7 5 5 5 5 5 29/30 Write block 71 len 12288 data: 5 5 6 30/30 Write block 451 len 40960 data: 5 7 7 6 5 5 5 3 3 3 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:21.984451272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 38 38 38 63 63 63 T F T <--- 4 080-099 38 38 38 63 63 63 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 38 38 38 63 63 63 F F F 7 140-159 40 38 40 65 63 65 F F F <--- 8 160-179 40 34 40 66 55 66 F F F <--- 9 180-199 40 38 40 67 63 67 F F F <--- 10 200-219 40 28 40 66 45 66 F F F <--- 11 220-239 40 31 40 68 51 68 F F F <--- 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 40 38 40 64 63 64 F T F <--- 15 300-319 40 38 40 64 63 64 F T F <--- 16 320-339 37 37 37 62 62 62 F F F 17 340-359 40 38 40 66 63 66 F F F <--- 18 360-379 40 36 40 66 61 66 F F F <--- 19 380-399 40 32 40 66 52 66 F F F <--- 20 400-419 40 38 40 64 63 64 F F F <--- 21 420-439 38 38 38 63 63 63 F F F 22 440-459 36 36 36 60 60 60 T F T <--- 23 460-479 40 32 40 65 52 65 T F T <--- 24 480-499 40 38 40 65 63 65 T F T <--- 25 500-519 37 37 37 62 62 62 F F F 26 520-539 40 37 40 64 62 64 F T F <--- 27 540-559 40 28 40 64 45 64 F T F <--- 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 40, Max flush: 68 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 41 41 41 69 69 69 F F F 4 080-099 38 38 38 63 63 63 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 38 38 38 63 63 63 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 67 67 67 F F F 10 200-219 40 40 40 66 66 66 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 40 40 40 64 64 64 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 40 40 40 66 66 66 F F F 18 360-379 40 40 40 66 66 66 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 40 40 40 64 64 64 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 41 41 41 69 69 69 F F F 24 480-499 41 41 41 69 69 69 F F F 25 500-519 37 37 37 62 62 62 F F F 26 520-539 40 40 40 64 64 64 F F F 27 540-559 40 40 40 64 64 64 F F F 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 41, Max flush: 69 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.315187503Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.326899543Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327223091Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327262188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327274648Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327292556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"Crucible d17a74a3-2b52-4702-b667-0572470144b8 has session id: 7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327607985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Upstairs opts: Upstairs UUID: d17a74a3-2b52-4702-b667-0572470144b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327643302Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Crucible stats registered with UUID: d17a74a3-2b52-4702-b667-0572470144b8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327659011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:26.32767226Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327869917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327895285Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.327910354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328046745Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328065773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"d17a74a3-2b52-4702-b667-0572470144b8 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328086762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328099871Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.3281137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328127819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328141958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328156007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328172986Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:51500)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328215293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:54590)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328353784Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:48817)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328381482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.328935664Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329003289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329112612Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329319258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329357985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[0]R flush_numbers[0..12]: [63, 54, 63, 69, 63, 63, 63, 65, 66, 67, 66, 68]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329473537Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[0]R generation[0..12]: [38, 34, 38, 41, 38, 38, 38, 40, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329490446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329507005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[1]R flush_numbers[0..12]: [63, 54, 63, 69, 63, 63, 63, 65, 66, 67, 66, 68]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329522804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[1]R generation[0..12]: [38, 34, 38, 41, 38, 38, 38, 40, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329538233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329557342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[2]R flush_numbers[0..12]: [63, 54, 63, 69, 63, 63, 63, 65, 66, 67, 66, 68]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329572751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[2]R generation[0..12]: [38, 34, 38, 41, 38, 38, 38, 40, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329588229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329604528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329624007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329637856Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329650845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329664744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"":"downstairs","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329678703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329691362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"0","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329705101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"1","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:33:26.329718991Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"client":"2","":"downstairs","session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329735709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329749168Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"d17a74a3-2b52-4702-b667-0572470144b8 is now active with session: 7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329764137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329778116Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368,"session_id":"7d44bfd8-f9c2-4d6f-bf7c-bd0621ba8f40"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329791456Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} {"msg":"Activated sub_volume d17a74a3-2b52-4702-b667-0572470144b8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:26.329802565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 32768 data: 6 6 4 4 4 2 2 3 02/30 Write block 227 len 36864 data: 7 8 7 7 7 5 4 5 5 03/30 Write block 548 len 8192 data: 3 3 04/30 Read block 438 len 12288 05/30 Write block 390 len 36864 data: 6 6 8 9 7 7 7 6 6 06/30 Write block 434 len 24576 data: 6 6 5 6 5 3 07/30 Write block 119 len 32768 data: 5 5 3 2 2 3 3 4 08/30 Write block 102 len 8192 data: 4 3 09/30 Read block 338 len 24576 10/30 Write block 73 len 16384 data: 7 6 6 5 11/30 Write block 520 len 4096 data: 4 12/30 Write block 486 len 40960 data: 5 7 8 8 6 6 6 6 6 5 13/30 Write block 179 len 8192 data: 5 6 14/30 Write block 396 len 24576 data: 8 7 7 5 5 4 15/30 Write block 426 len 40960 data: 8 8 8 8 7 7 6 5 7 7 16/30 Read block 442 len 12288 17/30 Read block 286 len 24576 18/30 Write block 393 len 32768 data: 10 8 8 9 8 8 6 6 19/30 Read block 429 len 40960 20/30 Read block 204 len 8192 21/30 Read block 331 len 20480 22/30 Read block 559 len 12288 23/30 Write block 450 len 28672 data: 5 6 8 8 7 6 7 24/30 Write block 232 len 28672 data: 6 5 6 6 4 3 2 25/30 Read block 121 len 24576 26/30 Read block 333 len 8192 27/30 Write block 468 len 20480 data: 4 4 4 4 3 28/30 Write block 351 len 36864 data: 5 5 5 6 6 7 7 7 7 29/30 Read block 409 len 16384 30/30 Write block 501 len 24576 data: 6 6 9 7 6 6 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:26.341275131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 41 41 41 69 69 69 T F T <--- 4 080-099 38 38 38 63 63 63 F F F 5 100-119 38 38 38 63 63 63 T F T <--- 6 120-139 38 38 38 63 63 63 T F T <--- 7 140-159 40 40 40 65 65 65 F F F 8 160-179 40 40 40 66 66 66 T F T <--- 9 180-199 40 40 40 67 67 67 T F T <--- 10 200-219 40 40 40 66 66 66 F F F 11 220-239 40 40 40 68 68 68 T T T 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 40 40 40 64 64 64 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 40 40 40 66 66 66 T F T <--- 18 360-379 40 40 40 66 66 66 F F F 19 380-399 40 40 40 66 66 66 T F T <--- 20 400-419 40 40 40 64 64 64 T F T <--- 21 420-439 38 38 38 63 63 63 T F T <--- 22 440-459 41 41 41 69 69 69 T T T 23 460-479 41 41 41 69 69 69 T T T 24 480-499 41 41 41 69 69 69 T F T <--- 25 500-519 37 37 37 62 62 62 T F T <--- 26 520-539 40 40 40 64 64 64 T F T <--- 27 540-559 40 40 40 64 64 64 T T T 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 41, Max flush: 69 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 63 63 63 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 38 38 38 63 63 63 F F F 3 060-079 42 42 42 70 70 70 F F F 4 080-099 38 38 38 63 63 63 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 42 42 42 70 70 70 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 42 42 42 70 70 70 F F F 9 180-199 42 42 42 70 70 70 F F F 10 200-219 40 40 40 66 66 66 F F F 11 220-239 42 42 42 70 70 70 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 40 40 40 64 64 64 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 42 42 42 70 70 70 F F F 18 360-379 40 40 40 66 66 66 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 42 42 42 70 70 70 F F F 21 420-439 42 42 42 70 70 70 F F F 22 440-459 42 42 42 70 70 70 F F F 23 460-479 42 42 42 70 70 70 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 36 36 36 61 61 61 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 42, Max flush: 70 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.677795537Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:30.688487186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.688537113Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.6989616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.69925992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699300117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699312466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699330245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"Crucible 26166586-177f-42c3-ab71-794f42393bb2 has session id: 06d1c949-60a4-4e3a-afe5-39a089af4a97","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699672352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Upstairs opts: Upstairs UUID: 26166586-177f-42c3-ab71-794f42393bb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699707209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Crucible stats registered with UUID: 26166586-177f-42c3-ab71-794f42393bb2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699722678Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:30.699736187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699894177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699930084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.699945763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700048766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700064815Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"26166586-177f-42c3-ab71-794f42393bb2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700082574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700095463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700109492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700124261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.70013847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700152869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700173098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:57927)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700212175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:45282)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700228094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:41269)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.700242703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701027249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701057587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701074806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701399974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701414983Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[0]R flush_numbers[0..12]: [63, 54, 63, 70, 63, 70, 70, 65, 70, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.70145047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[0]R generation[0..12]: [38, 34, 38, 42, 38, 42, 42, 40, 42, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701467199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701484028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[1]R flush_numbers[0..12]: [63, 54, 63, 70, 63, 70, 70, 65, 70, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701499717Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[1]R generation[0..12]: [38, 34, 38, 42, 38, 42, 42, 40, 42, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701515306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701532105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[2]R flush_numbers[0..12]: [63, 54, 63, 70, 63, 70, 70, 65, 70, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701547424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[2]R generation[0..12]: [38, 34, 38, 42, 38, 42, 42, 40, 42, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701562553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701578792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701592021Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.70160799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701621469Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701641037Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"":"downstairs","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701654926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701667655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"0","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701681595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"1","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701695244Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"client":"2","":"downstairs","session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701709293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701722652Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"26166586-177f-42c3-ab71-794f42393bb2 is now active with session: 06d1c949-60a4-4e3a-afe5-39a089af4a97","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.7017406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.70175374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376,"session_id":"06d1c949-60a4-4e3a-afe5-39a089af4a97"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701767279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} {"msg":"Activated sub_volume 26166586-177f-42c3-ab71-794f42393bb2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:30.701778798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 28672 data: 4 5 5 4 4 5 4 02/30 Write block 502 len 28672 data: 7 10 8 7 7 7 6 03/30 Read block 239 len 28672 04/30 Read block 264 len 16384 05/30 Read block 511 len 12288 06/30 Flush 07/30 Write block 18 len 32768 data: 7 7 7 5 5 6 6 3 08/30 Read block 38 len 36864 09/30 Read block 374 len 28672 10/30 Write block 165 len 36864 data: 7 7 6 5 5 4 5 7 7 11/30 Write block 312 len 24576 data: 4 4 5 4 3 3 12/30 Read block 164 len 36864 13/30 Write block 35 len 36864 data: 5 6 6 8 9 7 5 4 4 14/30 Write block 281 len 12288 data: 5 5 3 15/30 Read block 498 len 36864 16/30 Write block 297 len 32768 data: 4 4 4 4 5 6 6 7 17/30 Read block 170 len 20480 18/30 Read block 300 len 20480 19/30 Read block 16 len 36864 20/30 Read block 492 len 20480 21/30 Read block 142 len 24576 22/30 Read block 61 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 463 len 8192 data: 4 3 26/30 Write block 486 len 24576 data: 6 8 9 9 7 7 27/30 Write block 560 len 24576 data: 5 5 4 4 4 2 28/30 Write block 296 len 32768 data: 5 5 5 5 5 6 7 7 29/30 Write block 31 len 4096 data: 5 30/30 Read block 394 len 16384 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 0 2 25 Done 1 24 22 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:30.714758931Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 44 44 63 72 72 T F F <--- 1 020-039 34 44 44 54 72 72 T T T <--- 2 040-059 38 44 44 63 72 72 F F F <--- 3 060-079 44 44 44 71 71 71 F F F 4 080-099 38 38 38 63 63 63 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 42 42 42 70 70 70 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 42 44 44 70 72 72 F F F <--- 9 180-199 42 42 42 70 70 70 F F F 10 200-219 40 40 40 66 66 66 F F F 11 220-239 42 42 42 70 70 70 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 40 44 44 64 72 72 F T T <--- 15 300-319 40 44 44 64 72 72 F T T <--- 16 320-339 37 37 37 62 62 62 F F F 17 340-359 42 42 42 70 70 70 F F F 18 360-379 40 40 40 66 66 66 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 42 42 42 70 70 70 F F F 21 420-439 42 42 42 70 70 70 F F F 22 440-459 42 42 42 70 70 70 F F F 23 460-479 42 42 42 70 70 70 F T T <--- 24 480-499 42 42 42 70 70 70 F T T <--- 25 500-519 44 44 44 71 71 71 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 36 36 36 61 61 61 F T T <--- 29 580-599 37 37 37 62 62 62 F F F Max gen: 44, Max flush: 72 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 72 72 72 F F F 1 020-039 45 45 45 73 73 73 F F F 2 040-059 44 44 44 72 72 72 F F F 3 060-079 44 44 44 71 71 71 F F F 4 080-099 38 38 38 63 63 63 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 42 42 42 70 70 70 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 44 44 44 72 72 72 F F F 9 180-199 42 42 42 70 70 70 F F F 10 200-219 40 40 40 66 66 66 F F F 11 220-239 42 42 42 70 70 70 F F F 12 240-259 37 37 37 62 62 62 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 45 45 45 73 73 73 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 42 42 42 70 70 70 F F F 18 360-379 40 40 40 66 66 66 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 42 42 42 70 70 70 F F F 21 420-439 42 42 42 70 70 70 F F F 22 440-459 42 42 42 70 70 70 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 45 45 45 73 73 73 F F F 25 500-519 44 44 44 71 71 71 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 45, Max flush: 73 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.036960445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:35.04770979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.047765116Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058042784Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058402519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058439707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058454776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058472875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"Crucible 99ee4f62-02bf-46e3-99f4-4abd6af06968 has session id: 81be0f7e-cd5d-474d-9ff6-f924e674469c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058799162Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Upstairs opts: Upstairs UUID: 99ee4f62-02bf-46e3-99f4-4abd6af06968, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.05883819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Crucible stats registered with UUID: 99ee4f62-02bf-46e3-99f4-4abd6af06968","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.058853769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:35.058867498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059038056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059074304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059089942Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059202165Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059221643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"99ee4f62-02bf-46e3-99f4-4abd6af06968 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059252171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.05926628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059280529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059297178Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059311247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059325776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059339905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60949)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059387232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56891)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.059531412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33636)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.05955884Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060098943Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060133271Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060212246Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060478958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060520115Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[0]R flush_numbers[0..12]: [72, 73, 72, 71, 63, 70, 70, 65, 72, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.06058314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[0]R generation[0..12]: [44, 45, 44, 44, 38, 42, 42, 40, 44, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060601939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060618948Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[1]R flush_numbers[0..12]: [72, 73, 72, 71, 63, 70, 70, 65, 72, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060638007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[1]R generation[0..12]: [44, 45, 44, 44, 38, 42, 42, 40, 44, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060653476Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060670024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[2]R flush_numbers[0..12]: [72, 73, 72, 71, 63, 70, 70, 65, 72, 70, 66, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060686153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[2]R generation[0..12]: [44, 45, 44, 44, 38, 42, 42, 40, 44, 42, 40, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060702022Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060718671Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:33:35.06073279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060751769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060765088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060778677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"":"downstairs","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060792916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060805845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"0","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060820184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"1","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060834203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"client":"2","":"downstairs","session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060852602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060866131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"99ee4f62-02bf-46e3-99f4-4abd6af06968 is now active with session: 81be0f7e-cd5d-474d-9ff6-f924e674469c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.06088121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060893969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384,"session_id":"81be0f7e-cd5d-474d-9ff6-f924e674469c"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060907438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} {"msg":"Activated sub_volume 99ee4f62-02bf-46e3-99f4-4abd6af06968","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:35.060918897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 20480 data: 8 6 7 6 4 02/30 Write block 233 len 32768 data: 6 7 7 5 4 3 2 2 03/30 Write block 230 len 36864 data: 8 8 7 7 8 8 6 5 4 04/30 Write block 39 len 16384 data: 10 8 6 5 05/30 Read block 197 len 16384 06/30 Write block 5 len 24576 data: 5 4 4 4 4 5 07/30 Write block 61 len 24576 data: 4 4 5 6 6 5 08/30 Read block 280 len 36864 09/30 Write block 341 len 16384 data: 3 3 3 3 10/30 Read block 40 len 20480 11/30 Write block 201 len 24576 data: 5 6 7 8 7 8 12/30 Read block 0 len 36864 13/30 Write block 179 len 20480 data: 6 7 8 8 6 14/30 Write block 272 len 4096 data: 5 15/30 Read block 325 len 32768 16/30 Read block 406 len 4096 17/30 Write block 360 len 28672 data: 8 8 8 6 6 5 8 18/30 Flush 19/30 Write block 97 len 40960 data: 6 6 5 5 4 5 4 2 2 3 20/30 Read block 75 len 24576 21/30 Read block 24 len 32768 22/30 Read block 567 len 16384 23/30 Write block 414 len 12288 data: 6 6 7 24/30 Read block 93 len 32768 25/30 Write block 135 len 8192 data: 5 8 26/30 Write block 132 len 12288 data: 4 4 5 27/30 Write block 446 len 4096 data: 5 28/30 Read block 62 len 36864 29/30 Read block 271 len 32768 30/30 Write block 503 len 40960 data: 11 9 8 8 8 7 6 7 7 6 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:35.073369057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 46 46 72 74 74 F F F <--- 1 020-039 45 46 46 73 74 74 F F F <--- 2 040-059 44 46 46 72 74 74 F F F <--- 3 060-079 44 46 46 71 74 74 F F F <--- 4 080-099 38 38 38 63 63 63 F T T <--- 5 100-119 42 42 42 70 70 70 F T T <--- 6 120-139 42 42 42 70 70 70 F T T <--- 7 140-159 40 40 40 65 65 65 F F F 8 160-179 44 46 46 72 74 74 F F F <--- 9 180-199 42 46 46 70 74 74 F F F <--- 10 200-219 40 46 46 66 74 74 F F F <--- 11 220-239 42 46 46 70 74 74 T F F <--- 12 240-259 37 46 46 62 74 74 T F F <--- 13 260-279 36 46 46 61 74 74 F F F <--- 14 280-299 45 45 45 73 73 73 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 42 46 46 70 74 74 F F F <--- 18 360-379 40 46 46 66 74 74 F F F <--- 19 380-399 42 42 42 70 70 70 F F F 20 400-419 42 42 42 70 70 70 F T T <--- 21 420-439 42 46 46 70 74 74 T F F <--- 22 440-459 42 42 42 70 70 70 F T T <--- 23 460-479 45 45 45 73 73 73 F F F 24 480-499 45 45 45 73 73 73 F F F 25 500-519 44 44 44 71 71 71 F T T <--- 26 520-539 42 42 42 70 70 70 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 46, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 74 74 74 F F F 1 020-039 46 46 46 74 74 74 F F F 2 040-059 46 46 46 74 74 74 F F F 3 060-079 46 46 46 74 74 74 F F F 4 080-099 47 47 47 75 75 75 F F F 5 100-119 47 47 47 75 75 75 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 46 46 46 74 74 74 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 46 46 46 74 74 74 F F F 11 220-239 46 46 46 74 74 74 F F F 12 240-259 46 46 46 74 74 74 F F F 13 260-279 46 46 46 74 74 74 F F F 14 280-299 45 45 45 73 73 73 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 37 37 37 62 62 62 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 46 46 46 74 74 74 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 47 47 47 75 75 75 F F F 21 420-439 46 46 46 74 74 74 F F F 22 440-459 47 47 47 75 75 75 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 45 45 45 73 73 73 F F F 25 500-519 47 47 47 75 75 75 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 45 45 45 73 73 73 F F F 29 580-599 37 37 37 62 62 62 F F F Max gen: 47, Max flush: 75 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.428787131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.440615913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.440937461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.440973219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.440996587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441014296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"Crucible 46e15bce-3994-4961-9000-0f94c350282d has session id: cb199f9f-899e-4a0e-b24c-28a23651ab7c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441358352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Upstairs opts: Upstairs UUID: 46e15bce-3994-4961-9000-0f94c350282d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44139304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Crucible stats registered with UUID: 46e15bce-3994-4961-9000-0f94c350282d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441408949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:39.441425798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441577777Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441606555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441621374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441729397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441746956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"46e15bce-3994-4961-9000-0f94c350282d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441764545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441777374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441791473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441805832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44182838Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441842589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441856798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36329)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.441967151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50359)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.442093792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56397)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44212152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.442675322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44270298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.442728089Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443024028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443068775Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[0]R flush_numbers[0..12]: [74, 74, 74, 74, 75, 75, 75, 65, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443110953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[0]R generation[0..12]: [46, 46, 46, 46, 47, 47, 47, 40, 46, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443127901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44314488Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[1]R flush_numbers[0..12]: [74, 74, 74, 74, 75, 75, 75, 65, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443160569Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[1]R generation[0..12]: [46, 46, 46, 46, 47, 47, 47, 40, 46, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443175828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443192037Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[2]R flush_numbers[0..12]: [74, 74, 74, 74, 75, 75, 75, 65, 74, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443206996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[2]R generation[0..12]: [46, 46, 46, 46, 47, 47, 47, 40, 46, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443222225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443238464Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443254403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443275191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44328864Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443302249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"":"downstairs","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443316659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443329568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"0","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443343867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"1","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443357916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"client":"2","":"downstairs","session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443372175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443388764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"46e15bce-3994-4961-9000-0f94c350282d is now active with session: cb199f9f-899e-4a0e-b24c-28a23651ab7c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443403463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443416452Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392,"session_id":"cb199f9f-899e-4a0e-b24c-28a23651ab7c"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.443429871Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} {"msg":"Activated sub_volume 46e15bce-3994-4961-9000-0f94c350282d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:39.44344123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 24576 data: 9 9 8 8 7 6 02/30 Write block 382 len 40960 data: 4 4 4 4 4 5 6 7 7 7 03/30 Read block 560 len 40960 04/30 Write block 542 len 16384 data: 5 5 4 4 05/30 Read block 390 len 36864 06/30 Write block 9 len 8192 data: 5 6 07/30 Read block 407 len 32768 08/30 Flush 09/30 Write block 566 len 12288 data: 3 3 2 10/30 Write block 176 len 4096 data: 7 11/30 Write block 280 len 8192 data: 5 6 12/30 Read block 55 len 16384 13/30 Read block 180 len 12288 14/30 Read block 589 len 20480 15/30 Read block 282 len 4096 16/30 Write block 317 len 24576 data: 4 4 3 3 3 4 17/30 Write block 108 len 12288 data: 3 3 3 18/30 Read block 269 len 40960 19/30 Read block 331 len 4096 20/30 Read block 466 len 4096 21/30 Flush 22/30 Write block 570 len 12288 data: 2 3 4 23/30 Write block 217 len 40960 data: 5 5 5 6 5 6 6 6 6 6 24/30 Read block 398 len 40960 25/30 Read block 506 len 16384 26/30 Read block 334 len 24576 27/30 Write block 581 len 40960 data: 3 3 4 5 4 3 2 2 2 2 28/30 Write block 203 len 32768 data: 8 9 8 9 7 7 8 8 29/30 Read block 463 len 24576 30/30 Read block 453 len 16384 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:39.455404383Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 46 48 76 74 76 F F F <--- 1 020-039 46 46 46 74 74 74 F F F 2 040-059 46 46 46 74 74 74 F F F 3 060-079 46 46 46 74 74 74 F F F 4 080-099 47 47 47 75 75 75 F F F 5 100-119 48 47 48 77 75 77 F F F <--- 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 48 46 48 77 74 77 F F F <--- 9 180-199 46 46 46 74 74 74 F F F 10 200-219 46 46 46 74 74 74 T F T <--- 11 220-239 46 46 46 74 74 74 T F T <--- 12 240-259 46 46 46 74 74 74 F F F 13 260-279 46 46 46 74 74 74 F F F 14 280-299 48 45 48 77 73 77 F F F <--- 15 300-319 48 45 48 77 73 77 F F F <--- 16 320-339 48 37 48 77 62 77 F F F <--- 17 340-359 46 46 46 74 74 74 F F F 18 360-379 46 46 46 74 74 74 F F F 19 380-399 48 42 48 76 70 76 F F F <--- 20 400-419 47 47 47 75 75 75 F F F 21 420-439 48 46 48 76 74 76 F T F <--- 22 440-459 47 47 47 75 75 75 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 45 45 45 73 73 73 F F F 25 500-519 47 47 47 75 75 75 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 48 42 48 76 70 76 F F F <--- 28 560-579 48 45 48 77 73 77 T F T <--- 29 580-599 37 37 37 62 62 62 T F T <--- Max gen: 48, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 76 76 76 F F F 1 020-039 46 46 46 74 74 74 F F F 2 040-059 46 46 46 74 74 74 F F F 3 060-079 46 46 46 74 74 74 F F F 4 080-099 47 47 47 75 75 75 F F F 5 100-119 48 48 48 77 77 77 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 48 48 48 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 49 49 49 78 78 78 F F F 11 220-239 49 49 49 78 78 78 F F F 12 240-259 46 46 46 74 74 74 F F F 13 260-279 46 46 46 74 74 74 F F F 14 280-299 48 48 48 77 77 77 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 46 46 46 74 74 74 F F F 19 380-399 48 48 48 76 76 76 F F F 20 400-419 47 47 47 75 75 75 F F F 21 420-439 48 48 48 76 76 76 F F F 22 440-459 47 47 47 75 75 75 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 45 45 45 73 73 73 F F F 25 500-519 47 47 47 75 75 75 F F F 26 520-539 42 42 42 70 70 70 F F F 27 540-559 48 48 48 76 76 76 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F F F Max gen: 49, Max flush: 78 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.796313858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.80799447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.808312338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.808348895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.808362985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.808380923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"Crucible 038eedc2-8cfa-40df-9180-a462ff60ea8a has session id: 0aa821f9-0a92-4212-a03e-f8b9e0a1e103","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.80872014Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Upstairs opts: Upstairs UUID: 038eedc2-8cfa-40df-9180-a462ff60ea8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.808844912Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Crucible stats registered with UUID: 038eedc2-8cfa-40df-9180-a462ff60ea8a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.80887188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:43.808886129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809041718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809073776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809088825Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809187008Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809205707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"038eedc2-8cfa-40df-9180-a462ff60ea8a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809226696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809239645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809253474Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809269873Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809283502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809297731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.80931217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"ds_connection connected from Ok(127.0.0.1:33488)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809357257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"ds_connection connected from Ok(127.0.0.1:64960)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.809373705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"ds_connection connected from Ok(127.0.0.1:37351)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.80945507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810120034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810148213Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810224567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810488949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810526657Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[0]R flush_numbers[0..12]: [76, 74, 74, 74, 75, 77, 75, 65, 77, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810603631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[0]R generation[0..12]: [48, 46, 46, 46, 47, 48, 47, 40, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.81062038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810637589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[1]R flush_numbers[0..12]: [76, 74, 74, 74, 75, 77, 75, 65, 77, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810655828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[1]R generation[0..12]: [48, 46, 46, 46, 47, 48, 47, 40, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810671567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810687906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[2]R flush_numbers[0..12]: [76, 74, 74, 74, 75, 77, 75, 65, 77, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810702995Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[2]R generation[0..12]: [48, 46, 46, 46, 47, 48, 47, 40, 48, 46, 49, 49]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810718024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810733823Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810747152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.81076551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810778979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810792958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"":"downstairs","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810806858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810819477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"0","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810833306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"1","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810847325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"client":"2","":"downstairs","session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810865993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810879823Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"038eedc2-8cfa-40df-9180-a462ff60ea8a is now active with session: 0aa821f9-0a92-4212-a03e-f8b9e0a1e103","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810894972Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810907751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400,"session_id":"0aa821f9-0a92-4212-a03e-f8b9e0a1e103"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.81092084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} {"msg":"Activated sub_volume 038eedc2-8cfa-40df-9180-a462ff60ea8a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:43.810932329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 28672 data: 9 10 10 8 8 7 7 02/30 Write block 195 len 28672 data: 6 6 7 8 8 7 6 03/30 Read block 40 len 20480 04/30 Write block 494 len 40960 data: 7 6 3 4 6 7 7 7 8 12 05/30 Flush 06/30 Write block 395 len 20480 data: 9 10 9 9 7 07/30 Read block 252 len 32768 08/30 Write block 388 len 24576 data: 7 8 8 8 9 11 09/30 Read block 428 len 16384 10/30 Write block 448 len 24576 data: 5 4 6 7 9 9 11/30 Read block 152 len 32768 12/30 Write block 227 len 24576 data: 8 9 8 9 9 8 13/30 Read block 584 len 20480 14/30 Read block 41 len 36864 15/30 Read block 160 len 16384 16/30 Flush 17/30 Write block 503 len 32768 data: 13 10 9 9 9 8 7 8 18/30 Write block 79 len 16384 data: 3 3 4 3 19/30 Write block 526 len 16384 data: 3 4 4 4 20/30 Read block 195 len 4096 21/30 Write block 417 len 4096 data: 6 22/30 Read block 561 len 28672 23/30 Write block 88 len 4096 data: 3 24/30 Read block 296 len 4096 25/30 Read block 43 len 12288 26/30 Read block 589 len 36864 27/30 Read block 79 len 32768 28/30 Write block 374 len 16384 data: 3 5 5 6 29/30 Read block 189 len 12288 30/30 Read block 253 len 8192 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 7 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:43.823089498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 76 76 76 F F F 1 020-039 46 46 46 74 74 74 F F F 2 040-059 46 46 46 74 74 74 F F F 3 060-079 46 46 46 74 74 74 T T F <--- 4 080-099 47 47 47 75 75 75 T T F <--- 5 100-119 48 48 48 77 77 77 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 48 48 48 77 77 77 F F F 9 180-199 50 50 46 79 79 74 F F F <--- 10 200-219 50 50 49 79 79 78 F F F <--- 11 220-239 50 50 49 80 80 78 F F F <--- 12 240-259 46 46 46 74 74 74 F F F 13 260-279 46 46 46 74 74 74 F F F 14 280-299 48 48 48 77 77 77 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 46 46 46 74 74 74 T T F <--- 19 380-399 50 50 48 80 80 76 F F F <--- 20 400-419 47 47 47 75 75 75 T T F <--- 21 420-439 48 48 48 76 76 76 F F F 22 440-459 50 50 47 80 80 75 F F F <--- 23 460-479 45 45 45 73 73 73 F F F 24 480-499 50 50 45 79 79 73 F F T <--- 25 500-519 50 50 47 79 79 75 T T F <--- 26 520-539 42 42 42 70 70 70 T T F <--- 27 540-559 48 48 48 76 76 76 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F F F Max gen: 50, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 76 76 76 F F F 1 020-039 46 46 46 74 74 74 F F F 2 040-059 46 46 46 74 74 74 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 51 51 51 81 81 81 F F F 5 100-119 48 48 48 77 77 77 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 48 48 48 77 77 77 F F F 9 180-199 50 50 50 79 79 79 F F F 10 200-219 50 50 50 79 79 79 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 46 46 46 74 74 74 F F F 13 260-279 46 46 46 74 74 74 F F F 14 280-299 48 48 48 77 77 77 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 46 46 46 74 74 74 F F F 18 360-379 51 51 51 81 81 81 F F F 19 380-399 50 50 50 80 80 80 F F F 20 400-419 51 51 51 81 81 81 F F F 21 420-439 48 48 48 76 76 76 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 51 51 51 81 81 81 F F F 26 520-539 51 51 51 81 81 81 F F F 27 540-559 48 48 48 76 76 76 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F F F Max gen: 51, Max flush: 81 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.140425595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152124535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152454463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.15249338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152506189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152524428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"Crucible 701f23d6-b89e-43a0-a9f8-7edfbcc7da00 has session id: e83eaad7-548d-4ee2-baca-4072d59c440b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152842836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Upstairs opts: Upstairs UUID: 701f23d6-b89e-43a0-a9f8-7edfbcc7da00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152894063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Crucible stats registered with UUID: 701f23d6-b89e-43a0-a9f8-7edfbcc7da00","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.152910082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:48.152926521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153093309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153138066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153153805Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153259858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153277687Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"701f23d6-b89e-43a0-a9f8-7edfbcc7da00 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153308395Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153322044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153336163Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153350062Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153364141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.15337839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153392679Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33821)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153451595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"ds_connection connected from Ok(127.0.0.1:32953)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153487212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49462)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.153605994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154145187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154184115Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154282328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154536741Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154575268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[0]R flush_numbers[0..12]: [76, 74, 74, 81, 81, 77, 75, 65, 77, 79, 79, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154639724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[0]R generation[0..12]: [48, 46, 46, 51, 51, 48, 47, 40, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154656342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154673281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[1]R flush_numbers[0..12]: [76, 74, 74, 81, 81, 77, 75, 65, 77, 79, 79, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.15468873Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[1]R generation[0..12]: [48, 46, 46, 51, 51, 48, 47, 40, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154704539Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154720948Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[2]R flush_numbers[0..12]: [76, 74, 74, 81, 81, 77, 75, 65, 77, 79, 79, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154736497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[2]R generation[0..12]: [48, 46, 46, 51, 51, 48, 47, 40, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154751516Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154768225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154781604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Generation requested: 52 >= found:52","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-16T21:33:48.154803522Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154817261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154830861Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"":"downstairs","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.15484479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154857539Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"0","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154871428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"1","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154885407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"client":"2","":"downstairs","session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:33:48.154899926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154914675Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"701f23d6-b89e-43a0-a9f8-7edfbcc7da00 is now active with session: e83eaad7-548d-4ee2-baca-4072d59c440b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154929734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154942523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408,"session_id":"e83eaad7-548d-4ee2-baca-4072d59c440b"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154955762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} {"msg":"Activated sub_volume 701f23d6-b89e-43a0-a9f8-7edfbcc7da00","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:48.154967031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 17 len 32768 data: 7 8 8 8 6 6 7 7 03/30 Write block 502 len 4096 data: 9 04/30 Read block 421 len 36864 05/30 Write block 46 len 24576 data: 4 5 5 6 6 5 06/30 Write block 374 len 28672 data: 4 6 6 7 6 5 6 07/30 Read block 542 len 40960 08/30 Read block 484 len 24576 09/30 Write block 194 len 40960 data: 6 7 7 8 9 9 8 7 7 9 10/30 Write block 257 len 40960 data: 3 3 4 4 4 4 5 5 6 4 11/30 Write block 168 len 28672 data: 6 6 5 6 8 8 6 12/30 Write block 82 len 20480 data: 4 3 3 3 2 13/30 Read block 496 len 16384 14/30 Read block 244 len 12288 15/30 Read block 416 len 36864 16/30 Write block 399 len 8192 data: 8 7 17/30 Flush 18/30 Write block 514 len 40960 data: 5 6 6 7 7 4 5 3 3 2 19/30 Write block 61 len 28672 data: 5 5 6 7 7 6 5 20/30 Read block 33 len 36864 21/30 Write block 11 len 28672 data: 5 6 5 4 4 4 8 22/30 Write block 346 len 28672 data: 5 5 5 5 5 6 6 23/30 Write block 552 len 12288 data: 2 3 3 24/30 Write block 277 len 36864 data: 5 5 5 6 7 6 4 5 5 25/30 Read block 427 len 32768 26/30 Flush 27/30 Write block 501 len 36864 data: 8 10 14 11 10 10 10 9 8 28/30 Write block 470 len 16384 data: 5 5 4 3 29/30 Write block 216 len 36864 data: 5 6 6 6 7 6 7 7 7 30/30 Read block 572 len 12288 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 8 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:48.167750278Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 48 84 84 76 F F F <--- 1 020-039 52 52 46 83 83 74 F F F <--- 2 040-059 52 52 46 83 83 74 F F F <--- 3 060-079 52 52 51 84 84 81 F F F <--- 4 080-099 52 52 51 83 83 81 F F F <--- 5 100-119 48 48 48 77 77 77 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 52 52 48 83 83 77 F F F <--- 9 180-199 52 52 50 83 83 79 F F F <--- 10 200-219 52 52 50 83 83 79 T T F <--- 11 220-239 50 50 50 80 80 80 T T F <--- 12 240-259 52 52 46 83 83 74 F F F <--- 13 260-279 52 52 46 84 84 74 F F F <--- 14 280-299 52 52 48 84 84 77 F F F <--- 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 52 52 46 84 84 74 F F F <--- 18 360-379 52 52 51 83 83 81 F F F <--- 19 380-399 52 52 50 83 83 80 F F F <--- 20 400-419 52 52 51 83 83 81 F F F <--- 21 420-439 48 48 48 76 76 76 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 45 45 45 73 73 73 T T F <--- 24 480-499 50 50 50 79 79 79 F F F 25 500-519 52 52 51 84 84 81 T T F <--- 26 520-539 52 52 51 84 84 81 F F F <--- 27 540-559 52 52 48 84 84 76 F F F <--- 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F F F Max gen: 52, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 84 84 84 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 48 48 48 77 77 77 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 53 53 53 85 85 85 F F F 11 220-239 53 53 53 85 85 85 F F F 12 240-259 52 52 52 83 83 83 F F F 13 260-279 52 52 52 84 84 84 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 52 52 52 84 84 84 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 52 52 52 83 83 83 F F F 21 420-439 48 48 48 76 76 76 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 53 53 53 85 85 85 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 52 52 52 84 84 84 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F F F Max gen: 53, Max flush: 85 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.504379086Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:52.515103583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.515161659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.525588517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.525892026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.525930583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.525942843Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.525960471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"Crucible bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432 has session id: 51ec01b8-693e-4a4e-aff8-b22340fbc45c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526298968Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Upstairs opts: Upstairs UUID: bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526334376Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Crucible stats registered with UUID: bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526349755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:52.526363324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526524653Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526550131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.52656461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526672993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526690952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.52670669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.52671984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526733709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526747368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526761017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526775276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526808733Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63034)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526824802Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46181)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526839931Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42378)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.526963033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.52758842Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.527633737Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.527766378Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.527991993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528016241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[0]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 77, 75, 65, 83, 83, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528166991Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 52, 52, 48, 47, 40, 52, 52, 53, 53]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528187609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528204528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[1]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 77, 75, 65, 83, 83, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528219987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 52, 52, 48, 47, 40, 52, 52, 53, 53]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528235156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528251495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[2]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 77, 75, 65, 83, 83, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528266834Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 52, 52, 48, 47, 40, 52, 52, 53, 53]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528282243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528298762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:33:52.528312301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528331169Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528344109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528357698Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"":"downstairs","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528371877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528384356Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"0","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528400395Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"1","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528414234Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"client":"2","":"downstairs","session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528430603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528443562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432 is now active with session: 51ec01b8-693e-4a4e-aff8-b22340fbc45c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528458511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.5284713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416,"session_id":"51ec01b8-693e-4a4e-aff8-b22340fbc45c"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528484979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} {"msg":"Activated sub_volume bdd1d812-c8cb-44d2-9e74-e0cfdfcfa432","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:52.528496908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 242 len 28672 data: 6 6 6 6 6 5 5 02/30 Read block 14 len 8192 03/30 Write block 210 len 20480 data: 9 5 3 3 3 04/30 Read block 543 len 8192 05/30 Read block 105 len 4096 06/30 Write block 417 len 8192 data: 7 6 07/30 Read block 557 len 16384 08/30 Read block 248 len 40960 09/30 Read block 199 len 36864 10/30 Read block 313 len 28672 11/30 Read block 323 len 24576 12/30 Write block 378 len 40960 data: 7 6 7 5 5 5 5 5 5 6 13/30 Read block 385 len 28672 14/30 Read block 125 len 24576 15/30 Write block 580 len 32768 data: 2 4 4 5 6 5 4 3 16/30 Read block 353 len 36864 17/30 Read block 59 len 24576 18/30 Read block 283 len 16384 19/30 Read block 349 len 40960 20/30 Read block 290 len 32768 21/30 Read block 536 len 4096 22/30 Read block 74 len 8192 23/30 Read block 40 len 24576 24/30 Read block 105 len 4096 25/30 Read block 361 len 12288 26/30 Read block 347 len 16384 27/30 Write block 228 len 24576 data: 10 9 10 10 9 8 28/30 Write block 425 len 32768 data: 6 9 9 10 10 9 9 8 29/30 Write block 102 len 28672 data: 6 5 3 3 4 3 4 30/30 Write block 427 len 20480 data: 10 11 11 10 10 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 3 36 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:52.540442312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 84 84 84 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 48 48 48 77 77 77 F T T <--- 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 53 53 53 85 85 85 F T T <--- 11 220-239 53 53 53 85 85 85 F T T <--- 12 240-259 52 52 52 83 83 83 F T T <--- 13 260-279 52 52 52 84 84 84 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 52 52 52 84 84 84 F F F 18 360-379 52 52 52 83 83 83 F T T <--- 19 380-399 52 52 52 83 83 83 F T T <--- 20 400-419 52 52 52 83 83 83 F T T <--- 21 420-439 48 48 48 76 76 76 F T T <--- 22 440-459 50 50 50 80 80 80 F F F 23 460-479 53 53 53 85 85 85 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 52 52 52 84 84 84 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 49 49 49 78 78 78 F T T <--- Max gen: 53, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 84 84 84 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 54 54 54 86 86 86 F F F 6 120-139 47 47 47 75 75 75 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 54 54 54 86 86 86 F F F 11 220-239 54 54 54 86 86 86 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 52 52 52 84 84 84 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F F F 17 340-359 52 52 52 84 84 84 F F F 18 360-379 54 54 54 86 86 86 F F F 19 380-399 54 54 54 86 86 86 F F F 20 400-419 54 54 54 86 86 86 F F F 21 420-439 54 54 54 86 86 86 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 53 53 53 85 85 85 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 52 52 52 84 84 84 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 54, Max flush: 86 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.862532723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:56.873226482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.873285268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.883542988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.883838727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.883875855Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.883887924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.883905693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"Crucible f8c74651-10bd-4073-ba51-16987009690b has session id: 512b051f-dab9-4a12-92b5-7e61f91db07a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884250629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Upstairs opts: Upstairs UUID: f8c74651-10bd-4073-ba51-16987009690b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884288497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Crucible stats registered with UUID: f8c74651-10bd-4073-ba51-16987009690b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884304895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:56.884318695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884486313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884512251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.88452678Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884645842Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884664541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"f8c74651-10bd-4073-ba51-16987009690b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884685489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884698339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884712208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884726327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884739956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884754465Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884790902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59186)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884841919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42650)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884858598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59292)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.884936402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.885484985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.885513433Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.885684751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.885874808Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.885890527Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[0]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 86, 75, 65, 83, 83, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886079924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 52, 52, 54, 47, 40, 52, 52, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886097073Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886113722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[1]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 86, 75, 65, 83, 83, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886129101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 52, 52, 54, 47, 40, 52, 52, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.88614401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886160449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[2]R flush_numbers[0..12]: [84, 83, 83, 84, 83, 86, 75, 65, 83, 83, 86, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886178487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 52, 52, 54, 47, 40, 52, 52, 54, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886193996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886210275Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Max found gen is 55","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:33:56.886223704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886244023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886257262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886270261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"":"downstairs","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.88628427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886297099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"0","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886311208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"1","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-16T21:33:56.886325517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"client":"2","":"downstairs","session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886342926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886356255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"f8c74651-10bd-4073-ba51-16987009690b is now active with session: 512b051f-dab9-4a12-92b5-7e61f91db07a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886371244Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886384193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424,"session_id":"512b051f-dab9-4a12-92b5-7e61f91db07a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886399542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} {"msg":"Activated sub_volume f8c74651-10bd-4073-ba51-16987009690b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:33:56.886410992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 370 len 40960 data: 6 6 5 5 5 7 7 8 8 7 02/30 Write block 88 len 36864 data: 4 3 4 4 3 4 3 4 6 03/30 Read block 234 len 12288 04/30 Write block 164 len 8192 data: 6 8 05/30 Write block 478 len 8192 data: 6 6 06/30 Read block 356 len 20480 07/30 Read block 552 len 24576 08/30 Write block 215 len 12288 data: 4 6 7 09/30 Read block 247 len 32768 10/30 Read block 499 len 28672 11/30 Write block 129 len 4096 data: 5 12/30 Read block 546 len 8192 13/30 Write block 130 len 28672 data: 4 4 5 5 6 6 9 14/30 Read block 99 len 24576 15/30 Write block 386 len 40960 data: 6 7 8 9 9 9 10 12 9 10 16/30 Read block 473 len 12288 17/30 Flush 18/30 Write block 384 len 32768 data: 6 6 7 8 9 10 10 10 19/30 Write block 236 len 4096 data: 7 20/30 Read block 573 len 20480 21/30 Read block 391 len 12288 22/30 Read block 582 len 20480 23/30 Read block 244 len 36864 24/30 Read block 336 len 36864 25/30 Flush 26/30 Write block 547 len 8192 data: 3 4 27/30 Read block 166 len 36864 28/30 Write block 428 len 20480 data: 12 12 11 11 9 29/30 Write block 323 len 16384 data: 4 6 5 4 30/30 Read block 412 len 28672 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:33:56.899332899Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 84 84 84 F F F 4 080-099 52 56 56 83 87 87 F F F <--- 5 100-119 54 54 54 86 86 86 F F F 6 120-139 47 56 56 75 87 87 F F F <--- 7 140-159 40 40 40 65 65 65 F F F 8 160-179 52 56 56 83 87 87 F F F <--- 9 180-199 52 52 52 83 83 83 F F F 10 200-219 54 56 56 86 87 87 F F F <--- 11 220-239 54 56 56 86 88 88 F F F <--- 12 240-259 54 54 54 86 86 86 F F F 13 260-279 52 52 52 84 84 84 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 48 48 48 77 77 77 F T T <--- 17 340-359 52 52 52 84 84 84 F F F 18 360-379 54 56 56 86 87 87 F F F <--- 19 380-399 54 56 56 86 88 88 F F F <--- 20 400-419 54 54 54 86 86 86 F F F 21 420-439 54 54 54 86 86 86 F T T <--- 22 440-459 50 50 50 80 80 80 F F F 23 460-479 53 56 56 85 87 87 F F F <--- 24 480-499 50 50 50 79 79 79 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 52 52 52 84 84 84 F T T <--- 28 560-579 49 49 49 78 78 78 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 56, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 84 84 84 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 54 54 54 86 86 86 F F F 6 120-139 56 56 56 87 87 87 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 56 56 56 87 87 87 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 56 56 56 88 88 88 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 52 52 52 84 84 84 F F F 14 280-299 52 52 52 84 84 84 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 57 57 57 89 89 89 F F F 17 340-359 52 52 52 84 84 84 F F F 18 360-379 56 56 56 87 87 87 F F F 19 380-399 56 56 56 88 88 88 F F F 20 400-419 54 54 54 86 86 86 F F F 21 420-439 57 57 57 89 89 89 F F F 22 440-459 50 50 50 80 80 80 F F F 23 460-479 56 56 56 87 87 87 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 57 57 57 89 89 89 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 57, Max flush: 89 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.22568219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.237542129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.237879126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.237921463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.237934212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.237952481Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"Crucible c8964034-8ecf-47d4-849a-7824b9f3951a has session id: 8786b035-ee5b-4cf9-9cda-3aa384bb93d8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238278219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Upstairs opts: Upstairs UUID: c8964034-8ecf-47d4-849a-7824b9f3951a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238312567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Crucible stats registered with UUID: c8964034-8ecf-47d4-849a-7824b9f3951a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238328445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:01.238342035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238503734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238529322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238543861Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238654073Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238674112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"c8964034-8ecf-47d4-849a-7824b9f3951a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.23869679Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238709829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238723618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238737338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238751127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238765316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238779685Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54062)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238817772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41623)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238834061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40099)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.238933904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.239617187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.239644756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.239692072Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.24001664Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240065477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[0]R flush_numbers[0..12]: [84, 83, 83, 84, 87, 86, 87, 65, 87, 83, 87, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240147811Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 52, 56, 54, 56, 40, 56, 52, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.24016481Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240181829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[1]R flush_numbers[0..12]: [84, 83, 83, 84, 87, 86, 87, 65, 87, 83, 87, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240197618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 52, 56, 54, 56, 40, 56, 52, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240212797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240229306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[2]R flush_numbers[0..12]: [84, 83, 83, 84, 87, 86, 87, 65, 87, 83, 87, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240244265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 52, 56, 54, 56, 40, 56, 52, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240259494Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240275932Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2026-03-16T21:34:01.240289362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.2403095Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240323289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240336728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"":"downstairs","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240350757Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240365616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"0","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240379715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":"1","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240393444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"client":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2","":"downstairs","session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240411173Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240424512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"c8964034-8ecf-47d4-849a-7824b9f3951a is now active with session: 8786b035-ee5b-4cf9-9cda-3aa384bb93d8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240439511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.24045203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432,"session_id":"8786b035-ee5b-4cf9-9cda-3aa384bb93d8"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240465379Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} {"msg":"Activated sub_volume c8964034-8ecf-47d4-849a-7824b9f3951a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:01.240476679Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 16384 data: 9 11 10 11 02/30 Flush 03/30 Write block 547 len 8192 data: 4 5 04/30 Flush 05/30 Write block 214 len 8192 data: 4 5 06/30 Read block 371 len 8192 07/30 Read block 471 len 28672 08/30 Write block 349 len 16384 data: 6 6 7 7 09/30 Read block 283 len 8192 10/30 Read block 25 len 36864 11/30 Write block 355 len 28672 data: 7 8 8 8 8 9 9 12/30 Write block 269 len 36864 data: 3 4 5 6 5 5 5 4 6 13/30 Write block 268 len 40960 data: 3 4 5 6 7 6 6 6 5 7 14/30 Write block 60 len 40960 data: 4 6 6 7 8 8 7 6 6 5 15/30 Write block 427 len 20480 data: 11 13 13 12 12 16/30 Write block 207 len 28672 data: 8 8 9 10 6 4 4 17/30 Write block 287 len 20480 data: 4 6 5 5 5 18/30 Read block 350 len 20480 19/30 Write block 404 len 16384 data: 6 6 7 8 20/30 Write block 485 len 20480 data: 5 7 10 11 11 21/30 Read block 357 len 36864 22/30 Read block 169 len 24576 23/30 Read block 472 len 32768 24/30 Write block 409 len 4096 data: 7 25/30 Write block 210 len 4096 data: 11 26/30 Write block 323 len 4096 data: 5 27/30 Write block 353 len 36864 data: 6 7 8 9 9 9 9 10 10 28/30 Write block 454 len 12288 data: 8 7 8 29/30 Read block 139 len 4096 30/30 Write block 37 len 16384 data: 7 9 11 9 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 25 25 0 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:01.250828471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 52 52 52 83 83 83 T T F <--- 2 040-059 52 52 52 83 83 83 T T F <--- 3 060-079 52 52 52 84 84 84 T T F <--- 4 080-099 56 56 56 87 87 87 F F F 5 100-119 54 54 54 86 86 86 F F F 6 120-139 56 56 56 87 87 87 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 56 56 56 87 87 87 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 56 56 56 87 87 87 T T F <--- 11 220-239 58 58 58 90 90 90 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 52 52 52 84 84 84 T T F <--- 14 280-299 52 52 52 84 84 84 T T F <--- 15 300-319 48 48 48 77 77 77 F F F 16 320-339 57 57 57 89 89 89 T T F <--- 17 340-359 52 52 52 84 84 84 T T F <--- 18 360-379 56 56 56 87 87 87 T T F <--- 19 380-399 56 56 56 88 88 88 F F F 20 400-419 54 54 54 86 86 86 T T F <--- 21 420-439 57 57 57 89 89 89 T T F <--- 22 440-459 50 50 50 80 80 80 T T F <--- 23 460-479 56 56 56 87 87 87 F F F 24 480-499 50 50 50 79 79 79 T T F <--- 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 58 58 58 91 91 91 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 58, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 F F F 1 020-039 59 59 59 92 92 92 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 59 59 59 92 92 92 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 54 54 54 86 86 86 F F F 6 120-139 56 56 56 87 87 87 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 56 56 56 87 87 87 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 59 59 59 92 92 92 F F F 11 220-239 58 58 58 90 90 90 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 59 59 59 92 92 92 F F F 14 280-299 59 59 59 92 92 92 F F F 15 300-319 48 48 48 77 77 77 F F F 16 320-339 59 59 59 92 92 92 F F F 17 340-359 59 59 59 92 92 92 F F F 18 360-379 59 59 59 92 92 92 F F F 19 380-399 56 56 56 88 88 88 F F F 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 59 59 59 92 92 92 F F F 23 460-479 56 56 56 87 87 87 F F F 24 480-499 59 59 59 92 92 92 F F F 25 500-519 53 53 53 85 85 85 F F F 26 520-539 52 52 52 84 84 84 F F F 27 540-559 58 58 58 91 91 91 F F F 28 560-579 49 49 49 78 78 78 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 59, Max flush: 92 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.579057723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.590707407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591033535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591069513Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591081702Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591099471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"Crucible 65604a36-163e-4309-80c5-a8e8516565fb has session id: 5dfbdbbf-9309-48cb-9212-732fabe07ae6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591433138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Upstairs opts: Upstairs UUID: 65604a36-163e-4309-80c5-a8e8516565fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591470705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Crucible stats registered with UUID: 65604a36-163e-4309-80c5-a8e8516565fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591486404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:05.591499773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591670852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591706049Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591721628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59183904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591856219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"65604a36-163e-4309-80c5-a8e8516565fb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591876127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591888897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591902956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591917025Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591930714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591945073Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.591959262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54122)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.592002949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42861)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.592021788Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49407)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59213448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.592806224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59286763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.592886068Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593179128Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593206427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[0]R flush_numbers[0..12]: [84, 92, 92, 92, 87, 86, 87, 65, 87, 83, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593273012Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[0]R generation[0..12]: [52, 59, 59, 59, 56, 54, 56, 40, 56, 52, 59, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593290781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59330728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[1]R flush_numbers[0..12]: [84, 92, 92, 92, 87, 86, 87, 65, 87, 83, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593323039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[1]R generation[0..12]: [52, 59, 59, 59, 56, 54, 56, 40, 56, 52, 59, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593338368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593354216Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[2]R flush_numbers[0..12]: [84, 92, 92, 92, 87, 86, 87, 65, 87, 83, 92, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593369485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[2]R generation[0..12]: [52, 59, 59, 59, 56, 54, 56, 40, 56, 52, 59, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593387424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593403853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593422712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593437201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59345048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593463579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"":"downstairs","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593477788Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593490447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"0","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593504506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"1","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593518215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"client":"2","":"downstairs","session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:34:05.593532424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593548363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"65604a36-163e-4309-80c5-a8e8516565fb is now active with session: 5dfbdbbf-9309-48cb-9212-732fabe07ae6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593563472Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.593576251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440,"session_id":"5dfbdbbf-9309-48cb-9212-732fabe07ae6"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59358989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} {"msg":"Activated sub_volume 65604a36-163e-4309-80c5-a8e8516565fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:05.59360141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 344 len 32768 data: 4 3 6 6 6 7 7 8 02/30 Write block 516 len 28672 data: 7 8 8 5 6 4 4 03/30 Write block 326 len 32768 data: 5 4 4 2 2 2 3 3 04/30 Read block 479 len 16384 05/30 Write block 161 len 40960 data: 6 6 7 7 9 8 7 7 7 6 06/30 Flush 07/30 Flush 08/30 Write block 118 len 28672 data: 6 6 6 4 3 3 4 09/30 Flush 10/30 Write block 561 len 8192 data: 6 5 11/30 Read block 226 len 28672 12/30 Write block 297 len 24576 data: 6 6 6 6 7 8 13/30 Read block 425 len 4096 14/30 Read block 390 len 24576 15/30 Flush 16/30 Write block 527 len 20480 data: 5 5 5 5 4 17/30 Read block 406 len 36864 18/30 Flush 19/30 Write block 446 len 8192 data: 6 4 20/30 Write block 592 len 20480 data: 2 2 2 3 3 21/30 Read block 292 len 12288 22/30 Read block 500 len 16384 23/30 Read block 243 len 32768 24/30 Write block 593 len 8192 data: 3 3 25/30 Write block 227 len 28672 data: 10 12 11 12 11 10 9 26/30 Flush 27/30 Write block 499 len 36864 data: 8 8 9 11 15 12 11 11 11 28/30 Read block 357 len 4096 29/30 Read block 124 len 24576 30/30 Write block 9 len 36864 data: 6 7 6 7 6 5 5 5 9 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:05.606543195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 84 84 84 T T F <--- 1 020-039 59 59 59 92 92 92 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 59 59 59 92 92 92 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 60 60 54 95 95 86 F F F <--- 6 120-139 60 60 56 95 95 87 F F F <--- 7 140-159 40 40 40 65 65 65 F F F 8 160-179 60 60 56 93 93 87 F F F <--- 9 180-199 52 52 52 83 83 83 F F F 10 200-219 59 59 59 92 92 92 F F F 11 220-239 60 60 58 98 98 90 F F F <--- 12 240-259 54 54 54 86 86 86 F F F 13 260-279 59 59 59 92 92 92 F F F 14 280-299 60 60 59 96 96 92 F F F <--- 15 300-319 60 60 48 96 96 77 F F F <--- 16 320-339 60 60 59 93 93 92 F F F <--- 17 340-359 60 60 59 93 93 92 F F T <--- 18 360-379 59 59 59 92 92 92 F F F 19 380-399 56 56 56 88 88 88 F F F 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 60 60 59 98 98 92 F F F <--- 23 460-479 56 56 56 87 87 87 F F F 24 480-499 59 59 59 92 92 92 T T F <--- 25 500-519 60 60 53 93 93 85 T T T <--- 26 520-539 60 60 52 97 97 84 F F T <--- 27 540-559 58 58 58 91 91 91 F F F 28 560-579 60 60 49 96 96 78 F F F <--- 29 580-599 60 60 54 98 98 86 F F F <--- Max gen: 60, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 59 59 59 92 92 92 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 59 59 59 92 92 92 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 60 60 60 95 95 95 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 60 60 60 93 93 93 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 59 59 59 92 92 92 F F F 11 220-239 60 60 60 98 98 98 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 59 59 59 92 92 92 F F F 14 280-299 60 60 60 96 96 96 F F F 15 300-319 60 60 60 96 96 96 F F F 16 320-339 60 60 60 93 93 93 F F F 17 340-359 60 60 60 93 93 93 F F F 18 360-379 59 59 59 92 92 92 F F F 19 380-399 56 56 56 88 88 88 F F F 20 400-419 59 59 59 92 92 92 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 60 60 60 98 98 98 F F F 23 460-479 56 56 56 87 87 87 F F F 24 480-499 61 61 61 99 99 99 F F F 25 500-519 61 61 61 99 99 99 F F F 26 520-539 60 60 60 97 97 97 F F F 27 540-559 58 58 58 91 91 91 F F F 28 560-579 60 60 60 96 96 96 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 61, Max flush: 99 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.931474702Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943185442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943526699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943568336Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943581165Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943599214Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"Crucible 40869cb9-969b-4e36-8533-ff9f950ebbad has session id: 30001c7c-1e2e-4480-874d-e6b4059aaa6b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943894434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Upstairs opts: Upstairs UUID: 40869cb9-969b-4e36-8533-ff9f950ebbad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.943928761Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Crucible stats registered with UUID: 40869cb9-969b-4e36-8533-ff9f950ebbad","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.94394423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:09.943957889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944137007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944162005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944176374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944281267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944300036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"40869cb9-969b-4e36-8533-ff9f950ebbad active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944319515Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944332264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944346223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944360272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944373981Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.94438814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944402289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36069)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944451966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33195)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944480104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45775)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.944496263Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945215113Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945243322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.94526031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945583868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945608257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[0]R flush_numbers[0..12]: [99, 92, 92, 92, 87, 95, 95, 65, 93, 83, 92, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945626445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[0]R generation[0..12]: [61, 59, 59, 59, 56, 60, 60, 40, 60, 52, 59, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945642854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945659913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[1]R flush_numbers[0..12]: [99, 92, 92, 92, 87, 95, 95, 65, 93, 83, 92, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945675532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[1]R generation[0..12]: [61, 59, 59, 59, 56, 60, 60, 40, 60, 52, 59, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945690961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.94570757Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[2]R flush_numbers[0..12]: [99, 92, 92, 92, 87, 95, 95, 65, 93, 83, 92, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945723049Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[2]R generation[0..12]: [61, 59, 59, 59, 56, 60, 60, 40, 60, 52, 59, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945738228Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945754517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945767896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945787504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945801423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945814502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"":"downstairs","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945828882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945841581Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"0","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.94585743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"1","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945871429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"client":"2","":"downstairs","session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:34:09.945885708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945900667Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"40869cb9-969b-4e36-8533-ff9f950ebbad is now active with session: 30001c7c-1e2e-4480-874d-e6b4059aaa6b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945915616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945928375Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448,"session_id":"30001c7c-1e2e-4480-874d-e6b4059aaa6b"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945941754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} {"msg":"Activated sub_volume 40869cb9-969b-4e36-8533-ff9f950ebbad","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:09.945953063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 4096 data: 5 02/30 Write block 304 len 4096 data: 8 03/30 Read block 198 len 40960 04/30 Flush 05/30 Write block 395 len 32768 data: 11 11 10 10 9 8 5 6 06/30 Read block 464 len 40960 07/30 Read block 37 len 12288 08/30 Read block 56 len 20480 09/30 Write block 208 len 8192 data: 9 10 10/30 Write block 32 len 4096 data: 6 11/30 Read block 70 len 4096 12/30 Read block 107 len 16384 13/30 Write block 570 len 20480 data: 3 4 5 3 6 14/30 Write block 405 len 12288 data: 7 8 9 15/30 Write block 397 len 32768 data: 11 11 10 9 6 7 6 7 16/30 Flush 17/30 Write block 29 len 12288 data: 5 6 6 18/30 Read block 542 len 24576 19/30 Read block 145 len 16384 20/30 Write block 518 len 20480 data: 9 6 7 5 5 21/30 Write block 278 len 4096 data: 6 22/30 Write block 297 len 8192 data: 7 7 23/30 Flush 24/30 Write block 355 len 20480 data: 9 10 10 10 10 25/30 Write block 498 len 40960 data: 7 9 9 10 12 16 13 12 12 12 26/30 Read block 118 len 28672 27/30 Read block 350 len 8192 28/30 Write block 459 len 16384 data: 5 5 3 3 29/30 Read block 181 len 16384 30/30 Read block 181 len 12288 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:09.956941082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 62 59 62 102 92 102 F F F <--- 2 040-059 59 59 59 92 92 92 F F F 3 060-079 59 59 59 92 92 92 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 60 60 60 95 95 95 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 60 60 60 93 93 93 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 62 59 62 101 92 101 F F F <--- 11 220-239 60 60 60 98 98 98 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 62 59 62 102 92 102 F F F <--- 14 280-299 62 60 62 102 96 102 F F F <--- 15 300-319 62 60 62 100 96 100 F T F <--- 16 320-339 62 60 62 100 93 100 F T F <--- 17 340-359 60 60 60 93 93 93 T F T <--- 18 360-379 59 59 59 92 92 92 F F F 19 380-399 62 56 62 101 88 101 F F F <--- 20 400-419 62 59 62 101 92 101 F F F <--- 21 420-439 59 59 59 92 92 92 F F F 22 440-459 60 60 60 98 98 98 T F T <--- 23 460-479 56 56 56 87 87 87 T F T <--- 24 480-499 61 61 61 99 99 99 T F T <--- 25 500-519 62 61 62 102 99 102 T F T <--- 26 520-539 62 60 62 102 97 102 F F F <--- 27 540-559 58 58 58 91 91 91 F F F 28 560-579 62 60 62 101 96 101 F F F <--- 29 580-599 60 60 60 98 98 98 F F F Max gen: 62, Max flush: 102 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 F F F 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 59 59 59 92 92 92 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 60 60 60 95 95 95 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 60 60 60 93 93 93 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 62 62 62 101 101 101 F F F 11 220-239 60 60 60 98 98 98 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 62 62 62 100 100 100 F F F 16 320-339 62 62 62 100 100 100 F F F 17 340-359 63 63 63 103 103 103 F F F 18 360-379 59 59 59 92 92 92 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 62 62 62 101 101 101 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 63 63 63 103 103 103 F F F 23 460-479 63 63 63 103 103 103 F F F 24 480-499 63 63 63 103 103 103 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 62 62 62 102 102 102 F F F 27 540-559 58 58 58 91 91 91 F F F 28 560-579 62 62 62 101 101 101 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 63, Max flush: 103 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.2880158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.299661744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.299983952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30002138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300034239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300052408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"Crucible 52ace236-8081-439f-8b48-0874989f4e84 has session id: b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300411333Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Upstairs opts: Upstairs UUID: 52ace236-8081-439f-8b48-0874989f4e84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300445751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Crucible stats registered with UUID: 52ace236-8081-439f-8b48-0874989f4e84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300464369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:14.300477919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300646447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300672175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300687074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300798327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300816345Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"52ace236-8081-439f-8b48-0874989f4e84 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300833424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300846383Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300860812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300874821Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300930318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.300955756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301004233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"ds_connection connected from Ok(127.0.0.1:36754)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301053959Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"ds_connection connected from Ok(127.0.0.1:52849)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301085527Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"ds_connection connected from Ok(127.0.0.1:33440)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301102016Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301716214Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.301758721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30177593Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30206557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302109347Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[0]R flush_numbers[0..12]: [99, 102, 92, 92, 87, 95, 95, 65, 93, 83, 101, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302182442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[0]R generation[0..12]: [61, 62, 59, 59, 56, 60, 60, 40, 60, 52, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302199411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30221589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[1]R flush_numbers[0..12]: [99, 102, 92, 92, 87, 95, 95, 65, 93, 83, 101, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302231289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[1]R generation[0..12]: [61, 62, 59, 59, 56, 60, 60, 40, 60, 52, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302246478Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302263677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[2]R flush_numbers[0..12]: [99, 102, 92, 92, 87, 95, 95, 65, 93, 83, 101, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302278995Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[2]R generation[0..12]: [61, 62, 59, 59, 56, 60, 60, 40, 60, 52, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302297014Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302313373Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:34:14.302327292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302345481Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30235891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302372479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"":"downstairs","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302386498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302399187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"0","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302413196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"1","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302427145Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"client":"2","":"downstairs","session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302444144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302457623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"52ace236-8081-439f-8b48-0874989f4e84 is now active with session: b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302472642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.302485241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456,"session_id":"b5a7f6a1-c7c0-453e-a1a7-b61a5602cc17"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.30249888Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} {"msg":"Activated sub_volume 52ace236-8081-439f-8b48-0874989f4e84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:14.3025104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 313 len 36864 data: 5 6 5 4 5 5 4 4 4 02/30 Read block 576 len 24576 03/30 Read block 379 len 32768 04/30 Flush 05/30 Write block 220 len 32768 data: 8 7 8 8 8 7 7 11 06/30 Read block 544 len 40960 07/30 Read block 323 len 40960 08/30 Read block 294 len 12288 09/30 Read block 483 len 20480 10/30 Read block 506 len 28672 11/30 Read block 331 len 40960 12/30 Write block 67 len 28672 data: 7 7 6 5 6 6 8 13/30 Write block 204 len 24576 data: 10 9 10 9 10 11 14/30 Write block 161 len 32768 data: 7 7 8 8 10 9 8 8 15/30 Write block 556 len 20480 data: 4 4 5 5 6 16/30 Read block 441 len 40960 17/30 Read block 411 len 24576 18/30 Read block 257 len 8192 19/30 Write block 339 len 20480 data: 4 3 4 4 4 20/30 Write block 450 len 12288 data: 7 8 10 21/30 Write block 546 len 4096 data: 3 22/30 Flush 23/30 Write block 12 len 4096 data: 8 24/30 Read block 418 len 8192 25/30 Read block 142 len 32768 26/30 Read block 143 len 4096 27/30 Read block 241 len 8192 28/30 Write block 100 len 28672 data: 6 5 7 6 4 4 5 29/30 Read block 315 len 4096 30/30 Read block 593 len 20480 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:14.314671419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 99 99 99 T T F <--- 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 64 64 59 105 105 92 F F F <--- 4 080-099 56 56 56 87 87 87 F F F 5 100-119 60 60 60 95 95 95 T T F <--- 6 120-139 60 60 60 95 95 95 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 64 64 60 105 105 93 F F F <--- 9 180-199 52 52 52 83 83 83 F F F 10 200-219 64 64 62 105 105 101 F F F <--- 11 220-239 64 64 60 105 105 98 F F F <--- 12 240-259 54 54 54 86 86 86 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 62 104 104 100 F F F <--- 16 320-339 64 64 62 105 105 100 F F F <--- 17 340-359 64 64 63 105 105 103 F F F <--- 18 360-379 59 59 59 92 92 92 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 62 62 62 101 101 101 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 64 64 63 105 105 103 F F F <--- 23 460-479 63 63 63 103 103 103 F F F 24 480-499 63 63 63 103 103 103 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 62 62 62 102 102 102 F F F 27 540-559 64 64 58 105 105 91 F F F <--- 28 560-579 64 64 62 105 105 101 F F F <--- 29 580-599 60 60 60 98 98 98 F F F Max gen: 64, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 106 106 106 F F F 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 64 64 64 105 105 105 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 40 40 40 65 65 65 F F F 8 160-179 64 64 64 105 105 105 F F F 9 180-199 52 52 52 83 83 83 F F F 10 200-219 64 64 64 105 105 105 F F F 11 220-239 64 64 64 105 105 105 F F F 12 240-259 54 54 54 86 86 86 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 64 64 64 105 105 105 F F F 17 340-359 64 64 64 105 105 105 F F F 18 360-379 59 59 59 92 92 92 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 62 62 62 101 101 101 F F F 21 420-439 59 59 59 92 92 92 F F F 22 440-459 64 64 64 105 105 105 F F F 23 460-479 63 63 63 103 103 103 F F F 24 480-499 63 63 63 103 103 103 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 62 62 62 102 102 102 F F F 27 540-559 64 64 64 105 105 105 F F F 28 560-579 64 64 64 105 105 105 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 65, Max flush: 106 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.633769515Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.645450497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.645815632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.645856039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.645868668Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.645894267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"Crucible 2c2c9cf1-cfb3-4843-aab8-9921026a67e2 has session id: 4caaa177-b743-4479-bdea-55ba8b1c3a36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646193996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Upstairs opts: Upstairs UUID: 2c2c9cf1-cfb3-4843-aab8-9921026a67e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646221554Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Crucible stats registered with UUID: 2c2c9cf1-cfb3-4843-aab8-9921026a67e2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646236023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:18.646249132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64642704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646452038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646466538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646598718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"2c2c9cf1-cfb3-4843-aab8-9921026a67e2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646615517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646630206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646644475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"1","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646658284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646672003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646686392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646706801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"ds_connection connected from Ok(127.0.0.1:61515)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646731389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"ds_connection connected from Ok(127.0.0.1:47057)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646756068Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"ds_connection connected from Ok(127.0.0.1:37348)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.646790135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64745397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.647537444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.647618419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"1","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.647798736Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64789253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[0]R flush_numbers[0..12]: [106, 102, 92, 105, 87, 106, 95, 65, 105, 83, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.647986564Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[0]R generation[0..12]: [65, 62, 59, 64, 56, 65, 60, 40, 64, 52, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648006112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648022551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[1]R flush_numbers[0..12]: [106, 102, 92, 105, 87, 106, 95, 65, 105, 83, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64803831Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[1]R generation[0..12]: [65, 62, 59, 64, 56, 65, 60, 40, 64, 52, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648053709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648070048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[2]R flush_numbers[0..12]: [106, 102, 92, 105, 87, 106, 95, 65, 105, 83, 105, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648085297Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[2]R generation[0..12]: [65, 62, 59, 64, 56, 65, 60, 40, 64, 52, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648100336Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648116665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:34:18.648130454Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648149143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648162412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648175961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"":"downstairs","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.64819022Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648202809Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"0","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648218918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648236407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"client":"2","":"downstairs","session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648251046Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648264585Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"2c2c9cf1-cfb3-4843-aab8-9921026a67e2 is now active with session: 4caaa177-b743-4479-bdea-55ba8b1c3a36","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648279604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648291803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464,"session_id":"4caaa177-b743-4479-bdea-55ba8b1c3a36"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648304802Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} {"msg":"Activated sub_volume 2c2c9cf1-cfb3-4843-aab8-9921026a67e2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:18.648315991Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 36864 data: 7 8 7 8 8 9 10 8 8 02/30 Write block 554 len 24576 data: 4 4 5 5 6 6 03/30 Write block 143 len 36864 data: 4 6 5 3 3 4 5 5 5 04/30 Write block 356 len 24576 data: 11 11 11 11 11 11 05/30 Read block 450 len 12288 06/30 Read block 242 len 12288 07/30 Write block 194 len 36864 data: 7 8 8 9 10 10 9 8 8 08/30 Write block 245 len 8192 data: 7 7 09/30 Write block 487 len 28672 data: 11 12 12 9 9 8 8 10/30 Read block 552 len 12288 11/30 Read block 324 len 40960 12/30 Write block 429 len 8192 data: 14 13 13/30 Read block 37 len 40960 14/30 Read block 207 len 8192 15/30 Read block 432 len 40960 16/30 Write block 544 len 32768 data: 5 5 4 5 6 4 3 2 17/30 Flush 18/30 Write block 1 len 12288 data: 4 4 5 19/30 Read block 514 len 40960 20/30 Read block 327 len 8192 21/30 Write block 533 len 8192 data: 4 5 22/30 Read block 562 len 28672 23/30 Read block 134 len 12288 24/30 Read block 82 len 8192 25/30 Flush 26/30 Write block 456 len 28672 data: 9 7 5 6 6 4 4 27/30 Write block 553 len 28672 data: 4 5 5 6 6 7 7 28/30 Read block 5 len 40960 29/30 Read block 67 len 32768 30/30 Read block 579 len 36864 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:18.660791179Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 65 66 108 106 108 F F F <--- 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 64 64 64 105 105 105 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 40 66 107 65 107 F T F <--- 8 160-179 64 64 64 105 105 105 F F F 9 180-199 66 52 66 107 83 107 F F F <--- 10 200-219 66 64 66 107 105 107 F F F <--- 11 220-239 64 64 64 105 105 105 F F F 12 240-259 66 54 66 107 86 107 F F F <--- 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 64 64 64 105 105 105 F F F 17 340-359 66 64 66 107 105 107 F F F <--- 18 360-379 66 59 66 107 92 107 F F F <--- 19 380-399 62 62 62 101 101 101 F F F 20 400-419 66 62 66 107 101 107 F T F <--- 21 420-439 66 59 66 107 92 107 F F F <--- 22 440-459 64 64 64 105 105 105 T F T <--- 23 460-479 63 63 63 103 103 103 T F T <--- 24 480-499 66 63 66 107 103 107 F F F <--- 25 500-519 63 63 63 103 103 103 F F F 26 520-539 66 62 66 108 102 108 F F F <--- 27 540-559 66 64 66 107 105 107 T T T <--- 28 560-579 64 64 64 105 105 105 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 66, Max flush: 108 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 108 108 108 F F F 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 64 64 64 105 105 105 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 64 64 64 105 105 105 F F F 9 180-199 66 66 66 107 107 107 F F F 10 200-219 66 66 66 107 107 107 F F F 11 220-239 64 64 64 105 105 105 F F F 12 240-259 66 66 66 107 107 107 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 64 64 64 105 105 105 F F F 17 340-359 66 66 66 107 107 107 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 66 66 66 107 107 107 F F F 21 420-439 66 66 66 107 107 107 F F F 22 440-459 67 67 67 109 109 109 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 107 107 107 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 66 66 66 108 108 108 F F F 27 540-559 67 67 67 109 109 109 F F F 28 560-579 64 64 64 105 105 105 F F F 29 580-599 60 60 60 98 98 98 F F F Max gen: 67, Max flush: 109 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.9849282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.99663931Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.996959768Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997000425Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997012614Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997030463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"Crucible be77b9b3-2cc1-48c3-82a5-afe1c04756a1 has session id: da8de366-c4ec-43af-8a84-18c4f0037c1b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997375119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Upstairs opts: Upstairs UUID: be77b9b3-2cc1-48c3-82a5-afe1c04756a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997421806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Crucible stats registered with UUID: be77b9b3-2cc1-48c3-82a5-afe1c04756a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997437825Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:22.997451284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997632322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.99765781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997672719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997789471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.9978082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"be77b9b3-2cc1-48c3-82a5-afe1c04756a1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997828928Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997841678Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997855887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997869946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997883745Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997897934Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997912393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54717)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.997986588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55055)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.998016826Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47044)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.998033354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.998712698Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.998772814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.998801192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999100511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999142309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[0]R flush_numbers[0..12]: [108, 102, 92, 105, 87, 106, 95, 107, 105, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999229613Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[0]R generation[0..12]: [66, 62, 59, 64, 56, 65, 60, 66, 64, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999246162Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.99926291Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[1]R flush_numbers[0..12]: [108, 102, 92, 105, 87, 106, 95, 107, 105, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999278769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[1]R generation[0..12]: [66, 62, 59, 64, 56, 65, 60, 66, 64, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999294388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999310567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[2]R flush_numbers[0..12]: [108, 102, 92, 105, 87, 106, 95, 107, 105, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999326326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[2]R generation[0..12]: [66, 62, 59, 64, 56, 65, 60, 66, 64, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999344475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999360694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Max found gen is 68","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-03-16T21:34:22.999379712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999394041Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999407661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.99942125Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"":"downstairs","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999435359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999447878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"0","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999462107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"1","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msgDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999478786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"client":"2","":"downstairs","session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999493305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999506904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"be77b9b3-2cc1-48c3-82a5-afe1c04756a1 is now active with session: da8de366-c4ec-43af-8a84-18c4f0037c1b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999522083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999534402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472,"session_id":"da8de366-c4ec-43af-8a84-18c4f0037c1b"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.999547571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} {"msg":"Activated sub_volume be77b9b3-2cc1-48c3-82a5-afe1c04756a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:22.99956108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 36864 data: 2 2 3 5 5 6 7 6 5 02/30 Flush 03/30 Write block 445 len 32768 data: 5 7 5 6 5 8 9 11 04/30 Read block 223 len 12288 05/30 Write block 160 len 40960 data: 5 8 8 9 9 11 10 9 9 8 06/30 Flush 07/30 Write block 68 len 8192 data: 8 7 08/30 Read block 258 len 20480 09/30 Read block 40 len 28672 10/30 Write block 578 len 8192 data: 3 3 11/30 Read block 91 len 20480 12/30 Write block 327 len 40960 data: 6 5 3 3 3 4 4 3 3 4 13/30 Write block 160 len 16384 data: 6 9 9 10 14/30 Flush 15/30 Write block 437 len 28672 data: 8 7 5 4 4 4 5 16/30 Write block 421 len 36864 data: 6 5 6 7 7 10 12 14 15 17/30 Read block 367 len 4096 18/30 Read block 195 len 8192 19/30 Read block 456 len 20480 20/30 Read block 122 len 12288 21/30 Read block 539 len 16384 22/30 Write block 411 len 16384 data: 8 7 7 7 23/30 Read block 407 len 32768 24/30 Flush 25/30 Flush 26/30 Write block 570 len 40960 data: 4 5 6 4 7 6 5 4 4 4 27/30 Write block 524 len 40960 data: 2 3 4 6 6 6 6 5 4 5 28/30 Write block 579 len 4096 data: 5 29/30 Write block 331 len 12288 data: 4 5 5 30/30 Write block 224 len 12288 data: 9 8 8 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 28 36 Done 24 24 0 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:23.012083734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 108 108 108 F F F 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 68 68 64 112 112 105 F F F <--- 4 080-099 56 56 56 87 87 87 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 68 68 64 112 112 105 F F F <--- 9 180-199 66 66 66 107 107 107 F F F 10 200-219 66 66 66 107 107 107 F F F 11 220-239 64 64 64 105 105 105 T T F <--- 12 240-259 66 66 66 107 107 107 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 68 68 64 112 112 105 T T F <--- 17 340-359 66 66 66 107 107 107 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 66 113 113 107 F F F <--- 21 420-439 68 68 66 113 113 107 F F F <--- 22 440-459 68 68 67 113 113 109 F F F <--- 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 107 107 107 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 66 66 66 108 108 108 T T F <--- 27 540-559 67 67 67 109 109 109 F F F 28 560-579 68 68 68 112 112 110 T T F <--- 29 580-599 68 68 68 110 110 110 F F F Max gen: 68, Max flush: 113 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 108 108 108 F F F 1 020-039 62 62 62 102 102 102 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 68 68 68 112 112 112 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 68 68 68 112 112 112 F F F 9 180-199 66 66 66 107 107 107 F F F 10 200-219 66 66 66 107 107 107 F F F 11 220-239 69 69 69 114 114 114 F F F 12 240-259 66 66 66 107 107 107 F F F 13 260-279 62 62 62 102 102 102 F F F 14 280-299 62 62 62 102 102 102 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 66 66 66 107 107 107 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 68 68 68 113 113 113 F F F 22 440-459 68 68 68 113 113 113 F F F 23 460-479 67 67 67 109 109 109 F F F 24 480-499 66 66 66 107 107 107 F F F 25 500-519 63 63 63 103 103 103 F F F 26 520-539 69 69 69 114 114 114 F F F 27 540-559 67 67 67 109 109 109 F F F 28 560-579 69 69 69 114 114 114 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 69, Max flush: 114 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.329943945Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.341673944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.341986792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342034039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342046628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342071176Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"Crucible f0f26365-4a0d-41ec-967b-87203900970c has session id: 8eea07cb-86d4-49fe-9260-291b89622bf0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342405204Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Upstairs opts: Upstairs UUID: f0f26365-4a0d-41ec-967b-87203900970c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342439101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Crucible stats registered with UUID: f0f26365-4a0d-41ec-967b-87203900970c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34245451Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:27.342468019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342648717Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342684425Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342699874Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342816406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342833554Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"f0f26365-4a0d-41ec-967b-87203900970c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342855723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342868542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342882681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34289719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342911829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342926188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342940567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57166)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.342962646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41270)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.343120885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47245)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.343149313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.343740372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34378086Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.343798198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344103598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344150094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[0]R flush_numbers[0..12]: [108, 102, 92, 112, 87, 106, 95, 107, 112, 107, 107, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344200231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[0]R generation[0..12]: [66, 62, 59, 68, 56, 65, 60, 66, 68, 66, 66, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34421764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344235309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[1]R flush_numbers[0..12]: [108, 102, 92, 112, 87, 106, 95, 107, 112, 107, 107, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344251687Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[1]R generation[0..12]: [66, 62, 59, 68, 56, 65, 60, 66, 68, 66, 66, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344268066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344284355Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[2]R flush_numbers[0..12]: [108, 102, 92, 112, 87, 106, 95, 107, 112, 107, 107, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344300384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[2]R generation[0..12]: [66, 62, 59, 68, 56, 65, 60, 66, 68, 66, 66, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344316153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344333412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344347201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34436632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344379789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344393098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"":"downstairs","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344407057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344421626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"0","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344435605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"1","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344449344Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"client":"2","":"downstairs","session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"All required reconciliation work is completed","v":0,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "name":"crucible","level":30,"time":"2026-03-16T21:34:27.344466403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344479812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"f0f26365-4a0d-41ec-967b-87203900970c is now active with session: 8eea07cb-86d4-49fe-9260-291b89622bf0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344494701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.34450774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480,"session_id":"8eea07cb-86d4-49fe-9260-291b89622bf0"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344521099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} {"msg":"Activated sub_volume f0f26365-4a0d-41ec-967b-87203900970c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:27.344532668Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 28672 data: 3 4 4 5 5 4 4 02/30 Write block 243 len 16384 data: 7 7 8 8 03/30 Flush 04/30 Write block 200 len 40960 data: 10 9 9 10 11 10 11 10 11 12 05/30 Read block 292 len 28672 06/30 Write block 29 len 32768 data: 6 7 7 7 5 6 6 7 07/30 Write block 355 len 8192 data: 10 12 08/30 Read block 495 len 16384 09/30 Write block 470 len 24576 data: 6 6 5 4 3 3 10/30 Write block 437 len 40960 data: 9 8 6 5 5 5 6 4 6 8 11/30 Flush 12/30 Write block 555 len 28672 data: 6 7 7 8 8 7 7 13/30 Write block 535 len 40960 data: 3 3 3 3 3 5 5 6 6 6 14/30 Read block 32 len 12288 15/30 Write block 87 len 16384 data: 2 5 4 5 16/30 Write block 277 len 32768 data: 8 7 6 7 8 7 5 6 17/30 Read block 211 len 24576 18/30 Read block 262 len 24576 19/30 Flush 20/30 Write block 251 len 16384 data: 3 4 5 5 21/30 Read block 118 len 28672 22/30 Flush 23/30 Write block 60 len 20480 data: 5 7 7 8 9 24/30 Write block 477 len 16384 data: 3 7 7 8 25/30 Write block 436 len 24576 data: 7 10 9 7 6 6 26/30 Write block 170 len 24576 data: 7 7 9 9 7 8 27/30 Write block 261 len 16384 data: 5 5 6 6 28/30 Read block 303 len 28672 29/30 Write block 514 len 20480 data: 6 7 8 9 10 30/30 Read block 306 len 8192 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:27.357373891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 108 108 108 F F F 1 020-039 70 70 62 116 116 102 F F F <--- 2 040-059 59 59 59 92 92 92 F F F 3 060-079 70 70 70 115 115 115 T T F <--- 4 080-099 70 70 70 117 117 115 F F F <--- 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 68 68 68 112 112 112 T T F <--- 9 180-199 66 66 66 107 107 107 F F F 10 200-219 70 70 66 116 116 107 F F F <--- 11 220-239 69 69 69 114 114 114 F F F 12 240-259 70 70 70 118 118 115 F F F <--- 13 260-279 70 70 62 117 117 102 T T F <--- 14 280-299 70 70 62 117 117 102 F F F <--- 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 70 70 66 116 116 107 F F F <--- 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 70 70 68 116 116 113 T T F <--- 22 440-459 70 70 68 116 116 113 T T F <--- 23 460-479 70 70 67 116 116 109 T T F <--- 24 480-499 66 66 66 107 107 107 T T F <--- 25 500-519 63 63 63 103 103 103 T T F <--- 26 520-539 70 70 69 117 117 114 F F F <--- 27 540-559 70 70 67 117 117 109 F F F <--- 28 560-579 70 70 69 117 117 114 F F F <--- 29 580-599 68 68 68 110 110 110 F F F Max gen: 70, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 108 108 108 F F F 1 020-039 70 70 70 116 116 116 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 71 71 71 119 119 119 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 65 65 65 106 106 106 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 71 71 71 119 119 119 F F F 9 180-199 66 66 66 107 107 107 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 69 69 69 114 114 114 F F F 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 71 71 71 119 119 119 F F F 22 440-459 71 71 71 119 119 119 F F F 23 460-479 71 71 71 119 119 119 F F F 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 70 70 70 117 117 117 F F F 27 540-559 70 70 70 117 117 117 F F F 28 560-579 70 70 70 117 117 117 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 71, Max flush: 119 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.694932686Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:31.705601177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.705655293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716100629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716393619Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716433257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716445236Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716466164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"Crucible c746c53e-a177-4260-addd-615fb0745170 has session id: 686efb9c-a2e6-4a2e-bed2-977a12dbd4d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716780453Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Upstairs opts: Upstairs UUID: c746c53e-a177-4260-addd-615fb0745170, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716836399Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Crucible stats registered with UUID: c746c53e-a177-4260-addd-615fb0745170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.716851978Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:31.716865047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717031066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717056154Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717070613Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717194665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717212133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"c746c53e-a177-4260-addd-615fb0745170 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717228642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717241251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.71725545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717269729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717309827Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717359483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717374692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:62853)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717419719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58230)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717454807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61131)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.717470846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718126671Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718165848Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718183087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718529123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718588719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[0]R flush_numbers[0..12]: [108, 116, 92, 119, 117, 106, 95, 107, 119, 107, 116, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718651085Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[0]R generation[0..12]: [66, 70, 59, 71, 70, 65, 60, 66, 71, 66, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718669714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718686803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[1]R flush_numbers[0..12]: [108, 116, 92, 119, 117, 106, 95, 107, 119, 107, 116, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718702782Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[1]R generation[0..12]: [66, 70, 59, 71, 70, 65, 60, 66, 71, 66, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.71872097Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718737449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[2]R flush_numbers[0..12]: [108, 116, 92, 119, 117, 106, 95, 107, 119, 107, 116, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718752918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[2]R generation[0..12]: [66, 70, 59, 71, 70, 65, 60, 66, 71, 66, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718767887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718783996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718797315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718811344Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718829733Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718843362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"":"downstairs","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718857341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.71886996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"0","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718883959Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"1","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718897738Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"client":"2","":"downstairs","session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718911957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718925266Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"c746c53e-a177-4260-addd-615fb0745170 is now active with session: 686efb9c-a2e6-4a2e-bed2-977a12dbd4d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718945345Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718958134Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488,"session_id":"686efb9c-a2e6-4a2e-bed2-977a12dbd4d6"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718971423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} {"msg":"Activated sub_volume c746c53e-a177-4260-addd-615fb0745170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:31.718982902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 236 len 4096 data: 8 02/30 Read block 186 len 12288 03/30 Read block 448 len 24576 04/30 Write block 458 len 28672 data: 6 7 7 5 5 5 4 05/30 Read block 63 len 16384 06/30 Read block 321 len 36864 07/30 Read block 570 len 8192 08/30 Read block 375 len 40960 09/30 Write block 463 len 20480 data: 6 5 4 5 5 10/30 Read block 482 len 20480 11/30 Write block 8 len 36864 data: 5 7 8 7 9 7 6 6 6 12/30 Write block 77 len 12288 data: 5 4 5 13/30 Flush 14/30 Write block 549 len 28672 data: 5 4 3 3 5 6 7 15/30 Read block 584 len 40960 16/30 Read block 64 len 4096 17/30 Read block 442 len 24576 18/30 Write block 101 len 4096 data: 6 19/30 Write block 528 len 20480 data: 7 7 7 6 5 20/30 Read block 396 len 16384 21/30 Read block 127 len 40960 22/30 Read block 368 len 24576 23/30 Write block 64 len 32768 data: 10 9 8 8 9 8 6 7 24/30 Write block 188 len 24576 data: 3 4 4 4 6 6 25/30 Read block 563 len 28672 26/30 Read block 377 len 24576 27/30 Write block 471 len 28672 data: 7 6 5 4 4 3 4 28/30 Write block 6 len 20480 data: 5 5 6 8 9 29/30 Write block 234 len 24576 data: 9 9 9 6 5 3 30/30 Read block 131 len 4096 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:31.731422502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 72 72 108 120 120 F T T <--- 1 020-039 70 70 70 116 116 116 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 71 72 72 119 120 120 F T T <--- 4 080-099 70 70 70 117 117 117 F F F 5 100-119 65 65 65 106 106 106 F T T <--- 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 71 71 71 119 119 119 F F F 9 180-199 66 66 66 107 107 107 F T T <--- 10 200-219 70 70 70 116 116 116 F F F 11 220-239 69 72 72 114 120 120 T T T <--- 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 71 71 71 119 119 119 F F F 22 440-459 71 72 72 119 120 120 T F F <--- 23 460-479 71 72 72 119 120 120 T T T <--- 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 70 70 70 117 117 117 F T T <--- 27 540-559 70 70 70 117 117 117 F T T <--- 28 560-579 70 70 70 117 117 117 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 72, Max flush: 120 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 70 70 70 116 116 116 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 73 73 73 121 121 121 F F F 4 080-099 70 70 70 117 117 117 F F F 5 100-119 73 73 73 121 121 121 F F F 6 120-139 60 60 60 95 95 95 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 71 71 71 119 119 119 F F F 9 180-199 73 73 73 121 121 121 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 73 73 73 121 121 121 F F F 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 71 71 71 119 119 119 F F F 22 440-459 72 72 72 120 120 120 F F F 23 460-479 73 73 73 121 121 121 F F F 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 73 73 73 121 121 121 F F F 27 540-559 73 73 73 121 121 121 F F F 28 560-579 70 70 70 117 117 117 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 73, Max flush: 121 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.054907338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.066825724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067141512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.06717946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067191839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067209578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"Crucible 614f63cd-d05d-4445-bc34-5f65c63df28e has session id: 3629edee-381f-4e1b-9da5-343cd17e0b12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067537605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Upstairs opts: Upstairs UUID: 614f63cd-d05d-4445-bc34-5f65c63df28e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067583882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Crucible stats registered with UUID: 614f63cd-d05d-4445-bc34-5f65c63df28e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067599461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:36.06761306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067795758Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067837995Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067854054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067953637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067970976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"614f63cd-d05d-4445-bc34-5f65c63df28e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.067988675Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068001574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068015723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068030022Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068043801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.06805821Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068072759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:41581)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068133065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:48577)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068157773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:57777)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068173532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068918691Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.068959598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069026474Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069282496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069346332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[0]R flush_numbers[0..12]: [121, 116, 92, 121, 117, 121, 95, 107, 119, 121, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069431606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[0]R generation[0..12]: [73, 70, 59, 73, 70, 73, 60, 66, 71, 73, 70, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069449055Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069465644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[1]R flush_numbers[0..12]: [121, 116, 92, 121, 117, 121, 95, 107, 119, 121, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069481462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[1]R generation[0..12]: [73, 70, 59, 73, 70, 73, 60, 66, 71, 73, 70, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069496731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.06951307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[2]R flush_numbers[0..12]: [121, 116, 92, 121, 117, 121, 95, 107, 119, 121, 116, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069529059Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[2]R generation[0..12]: [73, 70, 59, 73, 70, 73, 60, 66, 71, 73, 70, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069544178Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069560047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069573426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:34:36.069587485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069608854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069622413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"":"downstairs","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069637042Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069650071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"0","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.06966441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"1","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069678349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"client":"2","":"downstairs","session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069692558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069705877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"614f63cd-d05d-4445-bc34-5f65c63df28e is now active with session: 3629edee-381f-4e1b-9da5-343cd17e0b12","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069723906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069736695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496,"session_id":"3629edee-381f-4e1b-9da5-343cd17e0b12"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069749974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} {"msg":"Activated sub_volume 614f63cd-d05d-4445-bc34-5f65c63df28e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:36.069761603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 36864 data: 8 9 11 10 9 9 10 9 7 02/30 Read block 448 len 40960 03/30 Read block 52 len 24576 04/30 Write block 118 len 28672 data: 7 7 7 5 4 4 5 05/30 Read block 86 len 12288 06/30 Write block 27 len 36864 data: 4 4 7 8 8 8 6 7 7 07/30 Read block 321 len 8192 08/30 Flush 09/30 Write block 225 len 12288 data: 9 9 12 10/30 Read block 413 len 40960 11/30 Read block 98 len 24576 12/30 Read block 33 len 36864 13/30 Write block 556 len 12288 data: 8 8 9 14/30 Read block 226 len 32768 15/30 Flush 16/30 Write block 341 len 40960 data: 5 5 5 5 4 7 7 7 8 8 17/30 Read block 482 len 40960 18/30 Flush 19/30 Write block 165 len 4096 data: 12 20/30 Read block 521 len 40960 21/30 Read block 345 len 16384 22/30 Write block 432 len 12288 data: 10 7 8 23/30 Read block 141 len 32768 24/30 Flush 25/30 Write block 136 len 12288 data: 10 7 8 26/30 Write block 438 len 4096 data: 10 27/30 Read block 250 len 4096 28/30 Read block 238 len 32768 29/30 Read block 392 len 36864 30/30 Write block 76 len 24576 data: 6 6 5 6 5 6 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 23 1 25 Done 29 7 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:36.082923124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 74 70 74 122 116 122 F T F <--- 2 040-059 59 59 59 92 92 92 F F F 3 060-079 74 73 74 122 121 122 T T T <--- 4 080-099 70 70 70 117 117 117 T F T <--- 5 100-119 74 73 74 122 121 122 F T F <--- 6 120-139 74 60 74 122 95 122 T T T <--- 7 140-159 66 66 66 107 107 107 F F F 8 160-179 74 71 74 125 119 125 F F F <--- 9 180-199 73 73 73 121 121 121 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 73 74 123 121 123 F F F <--- 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 74 70 74 124 116 124 F F F <--- 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 74 71 74 125 119 125 T F T <--- 22 440-459 72 72 72 120 120 120 F F F 23 460-479 73 73 73 121 121 121 F F F 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 73 73 73 121 121 121 F F F 27 540-559 74 73 74 123 121 123 F F F <--- 28 560-579 70 70 70 117 117 117 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 74, Max flush: 125 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 74 74 74 122 122 122 F F F 2 040-059 59 59 59 92 92 92 F F F 3 060-079 75 75 75 126 126 126 F F F 4 080-099 75 75 75 126 126 126 F F F 5 100-119 74 74 74 122 122 122 F F F 6 120-139 75 75 75 126 126 126 F F F 7 140-159 66 66 66 107 107 107 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 73 73 73 121 121 121 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 74 74 123 123 123 F F F 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 64 64 104 104 104 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 74 74 74 124 124 124 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 75 75 75 126 126 126 F F F 22 440-459 72 72 72 120 120 120 F F F 23 460-479 73 73 73 121 121 121 F F F 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 73 73 73 121 121 121 F F F 27 540-559 74 74 74 123 123 123 F F F 28 560-579 70 70 70 117 117 117 F F F 29 580-599 68 68 68 110 110 110 F F F Max gen: 75, Max flush: 126 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.400460847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:40.411267318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.411319225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.421820897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422137626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422176243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422188402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422206161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"Crucible c249fa53-8081-4588-84d5-1ad4871840d6 has session id: 80200014-d8ed-412b-a19a-795c99a17161","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422531119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Upstairs opts: Upstairs UUID: c249fa53-8081-4588-84d5-1ad4871840d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422566916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Crucible stats registered with UUID: c249fa53-8081-4588-84d5-1ad4871840d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422582705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:40.422596314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422772362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.42280837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422823919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422914413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422932051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"c249fa53-8081-4588-84d5-1ad4871840d6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.42294997Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422962859Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.422992467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423006946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423021045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423035404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423049373Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"ds_connection connected from Ok(127.0.0.1:44770)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423064262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"ds_connection connected from Ok(127.0.0.1:52718)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423079371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"ds_connection connected from Ok(127.0.0.1:42110)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.4230999Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423854378Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423890796Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.423935223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424226113Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.4242681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[0]R flush_numbers[0..12]: [121, 122, 92, 126, 126, 122, 126, 107, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424346145Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[0]R generation[0..12]: [73, 74, 59, 75, 75, 74, 75, 66, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424362774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424382522Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[1]R flush_numbers[0..12]: [121, 122, 92, 126, 126, 122, 126, 107, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424398761Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[1]R generation[0..12]: [73, 74, 59, 75, 75, 74, 75, 66, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424414Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424430439Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[2]R flush_numbers[0..12]: [121, 122, 92, 126, 126, 122, 126, 107, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424446028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[2]R generation[0..12]: [73, 74, 59, 75, 75, 74, 75, 66, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424461457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424477436Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424490925Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424511283Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424524743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424537962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"":"downstairs","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424551701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.42456387Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"0","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424578059Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"1","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424592048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"client":"2","":"downstairs","session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424607927Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424624166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"c249fa53-8081-4588-84d5-1ad4871840d6 is now active with session: 80200014-d8ed-412b-a19a-795c99a17161","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424638985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424651644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504,"session_id":"80200014-d8ed-412b-a19a-795c99a17161"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424665413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} {"msg":"Activated sub_volume c249fa53-8081-4588-84d5-1ad4871840d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:40.424677212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 310 len 24576 data: 5 5 5 6 7 6 02/30 Read block 204 len 40960 03/30 Flush 04/30 Write block 133 len 40960 data: 6 7 7 11 8 9 7 7 6 4 05/30 Read block 580 len 16384 06/30 Read block 465 len 20480 07/30 Read block 337 len 24576 08/30 Read block 363 len 8192 09/30 Write block 59 len 8192 data: 3 6 10/30 Read block 408 len 16384 11/30 Write block 90 len 32768 data: 6 5 4 5 4 5 7 7 12/30 Write block 451 len 40960 data: 10 12 10 9 8 10 8 7 8 8 13/30 Flush 14/30 Write block 346 len 8192 data: 8 8 15/30 Read block 403 len 20480 16/30 Write block 155 len 16384 data: 5 5 5 5 17/30 Read block 122 len 40960 18/30 Read block 446 len 24576 19/30 Read block 231 len 24576 20/30 Write block 111 len 8192 data: 4 5 21/30 Write block 597 len 4096 data: 2 22/30 Write block 561 len 32768 data: 8 6 5 5 3 4 4 3 23/30 Write block 21 len 20480 data: 7 7 8 8 4 24/30 Read block 337 len 40960 25/30 Read block 428 len 40960 26/30 Read block 478 len 36864 27/30 Read block 536 len 40960 28/30 Read block 422 len 20480 29/30 Read block 412 len 8192 30/30 Read block 450 len 16384 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:40.437118442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 74 74 74 122 122 122 F T T <--- 2 040-059 59 76 76 92 128 128 F F F <--- 3 060-079 75 76 76 126 128 128 F F F <--- 4 080-099 75 76 76 126 128 128 F F F <--- 5 100-119 74 74 74 122 122 122 F T T <--- 6 120-139 75 76 76 126 128 128 F F F <--- 7 140-159 66 76 76 107 128 128 F T T <--- 8 160-179 74 74 74 125 125 125 F F F 9 180-199 73 73 73 121 121 121 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 74 74 123 123 123 F F F 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 64 76 76 104 127 127 F F F <--- 16 320-339 69 69 69 114 114 114 F F F 17 340-359 74 74 74 124 124 124 F T T <--- 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 75 75 75 126 126 126 F F F 22 440-459 72 76 76 120 128 128 F F F <--- 23 460-479 73 76 76 121 128 128 F F F <--- 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 73 73 73 121 121 121 F F F 27 540-559 74 74 74 123 123 123 F F F 28 560-579 70 70 70 117 117 117 F T T <--- 29 580-599 68 68 68 110 110 110 F T T <--- Max gen: 76, Max flush: 128 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 77 77 77 129 129 129 F F F 2 040-059 76 76 76 128 128 128 F F F 3 060-079 76 76 76 128 128 128 F F F 4 080-099 76 76 76 128 128 128 F F F 5 100-119 77 77 77 129 129 129 F F F 6 120-139 76 76 76 128 128 128 F F F 7 140-159 77 77 77 129 129 129 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 73 73 73 121 121 121 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 74 74 123 123 123 F F F 12 240-259 70 70 70 118 118 118 F F F 13 260-279 71 71 71 119 119 119 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 76 76 76 127 127 127 F F F 16 320-339 69 69 69 114 114 114 F F F 17 340-359 77 77 77 129 129 129 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 75 75 75 126 126 126 F F F 22 440-459 76 76 76 128 128 128 F F F 23 460-479 76 76 76 128 128 128 F F F 24 480-499 71 71 71 119 119 119 F F F 25 500-519 71 71 71 119 119 119 F F F 26 520-539 73 73 73 121 121 121 F F F 27 540-559 74 74 74 123 123 123 F F F 28 560-579 77 77 77 129 129 129 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 77, Max flush: 129 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.769829048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:44.780522357Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.780575844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.790860081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791161351Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791200818Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791212867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791230756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"Crucible f827d297-733a-4db3-895c-8b08359b317d has session id: fa2313d1-d482-4804-a0fb-1aedac5cbe84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791567543Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Upstairs opts: Upstairs UUID: f827d297-733a-4db3-895c-8b08359b317d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791627569Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Crucible stats registered with UUID: f827d297-733a-4db3-895c-8b08359b317d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791644517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:44.791657847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791820435Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791845514Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791860263Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.791991284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792009603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"f827d297-733a-4db3-895c-8b08359b317d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.79203971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.7920542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792068719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792082798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792129054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792158982Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792174211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"ds_connection connected from Ok(127.0.0.1:37830)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792276664Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"ds_connection connected from Ok(127.0.0.1:62270)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792305122Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"ds_connection connected from Ok(127.0.0.1:49185)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792321191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.792971847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793013354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793030622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793349471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793396017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[0]R flush_numbers[0..12]: [121, 129, 128, 128, 128, 129, 128, 129, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793454183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[0]R generation[0..12]: [73, 77, 76, 76, 76, 77, 76, 77, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793470922Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793487511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[1]R flush_numbers[0..12]: [121, 129, 128, 128, 128, 129, 128, 129, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.79350342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[1]R generation[0..12]: [73, 77, 76, 76, 76, 77, 76, 77, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793519019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793535348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[2]R flush_numbers[0..12]: [121, 129, 128, 128, 128, 129, 128, 129, 125, 121, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793550987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[2]R generation[0..12]: [73, 77, 76, 76, 76, 77, 76, 77, 74, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793568586Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793584745Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-16T21:34:44.793598034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793618152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793631501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.79364502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"":"downstairs","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793659289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793672109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"0","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793686208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"1","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793699997Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"client":"2","":"downstairs","session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"All required reconciliation work is completed",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793716926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793730255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"f827d297-733a-4db3-895c-8b08359b317d is now active with session: fa2313d1-d482-4804-a0fb-1aedac5cbe84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793745034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793757913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512,"session_id":"fa2313d1-d482-4804-a0fb-1aedac5cbe84"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793771412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} {"msg":"Activated sub_volume f827d297-733a-4db3-895c-8b08359b317d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:44.793782601Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 32768 data: 7 5 6 6 5 5 5 5 02/30 Write block 438 len 4096 data: 11 03/30 Write block 507 len 24576 data: 13 10 9 9 8 7 04/30 Flush 05/30 Write block 420 len 28672 data: 7 7 6 7 8 8 11 06/30 Write block 531 len 8192 data: 7 6 07/30 Read block 521 len 16384 08/30 Read block 222 len 40960 09/30 Write block 249 len 24576 data: 5 5 4 5 6 6 10/30 Write block 493 len 16384 data: 9 8 7 4 11/30 Read block 525 len 16384 12/30 Read block 349 len 40960 13/30 Read block 68 len 32768 14/30 Read block 232 len 20480 15/30 Read block 549 len 28672 16/30 Write block 38 len 28672 data: 10 12 10 7 6 5 4 17/30 Read block 140 len 20480 18/30 Read block 168 len 36864 19/30 Write block 464 len 32768 data: 6 5 6 6 5 5 7 8 20/30 Read block 570 len 40960 21/30 Write block 55 len 20480 data: 2 3 4 4 4 22/30 Write block 73 len 4096 data: 9 23/30 Flush 24/30 Write block 491 len 32768 data: 10 9 10 9 8 5 5 8 25/30 Write block 255 len 28672 data: 2 3 4 4 5 5 6 26/30 Write block 320 len 16384 data: 6 6 6 6 27/30 Write block 138 len 4096 data: 10 28/30 Read block 186 len 20480 29/30 Write block 335 len 20480 data: 4 5 5 4 5 30/30 Write block 182 len 40960 data: 9 7 6 6 5 5 4 5 5 5 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:44.805991147Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 77 78 78 129 131 131 F F F <--- 2 040-059 76 78 78 128 131 131 F F F <--- 3 060-079 76 78 78 128 131 131 F F F <--- 4 080-099 76 76 76 128 128 128 F F F 5 100-119 77 77 77 129 129 129 F F F 6 120-139 76 76 76 128 128 128 F T T <--- 7 140-159 77 77 77 129 129 129 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 73 73 73 121 121 121 F T T <--- 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 74 74 123 123 123 F F F 12 240-259 70 78 78 118 131 131 F T T <--- 13 260-279 71 71 71 119 119 119 F T T <--- 14 280-299 70 70 70 117 117 117 F F F 15 300-319 76 78 78 127 130 130 T F F <--- 16 320-339 69 78 78 114 130 130 T T T <--- 17 340-359 77 77 77 129 129 129 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 75 78 78 126 131 131 F F F <--- 22 440-459 76 76 76 128 128 128 F F F 23 460-479 76 78 78 128 131 131 F F F <--- 24 480-499 71 78 78 119 131 131 F T T <--- 25 500-519 71 78 78 119 130 130 F F F <--- 26 520-539 73 78 78 121 131 131 F F F <--- 27 540-559 74 74 74 123 123 123 F F F 28 560-579 77 77 77 129 129 129 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 78, Max flush: 131 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 78 78 78 131 131 131 F F F 2 040-059 78 78 78 131 131 131 F F F 3 060-079 78 78 78 131 131 131 F F F 4 080-099 76 76 76 128 128 128 F F F 5 100-119 77 77 77 129 129 129 F F F 6 120-139 79 79 79 132 132 132 F F F 7 140-159 77 77 77 129 129 129 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 79 79 79 132 132 132 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 74 74 74 123 123 123 F F F 12 240-259 79 79 79 132 132 132 F F F 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 78 78 78 130 130 130 F F F 16 320-339 79 79 79 132 132 132 F F F 17 340-359 77 77 77 129 129 129 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 62 62 62 101 101 101 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 78 78 78 131 131 131 F F F 22 440-459 76 76 76 128 128 128 F F F 23 460-479 78 78 78 131 131 131 F F F 24 480-499 79 79 79 132 132 132 F F F 25 500-519 78 78 78 130 130 130 F F F 26 520-539 78 78 78 131 131 131 F F F 27 540-559 74 74 74 123 123 123 F F F 28 560-579 77 77 77 129 129 129 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 79, Max flush: 132 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.150807036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.162650656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.162964475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163016281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.1630295Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163047839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"Crucible b50a6796-8556-4d2f-b172-ecc1c7eef011 has session id: 0a725065-01b6-4573-9482-300f3dc0c463","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163337129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Upstairs opts: Upstairs UUID: b50a6796-8556-4d2f-b172-ecc1c7eef011, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163373337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Crucible stats registered with UUID: b50a6796-8556-4d2f-b172-ecc1c7eef011","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163387936Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:49.163400955Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163573923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163610471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.16362605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163732722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163752841Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"b50a6796-8556-4d2f-b172-ecc1c7eef011 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.16377211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163784979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163798768Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163812927Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163826506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163840735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163854654Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"ds_connection connected from Ok(127.0.0.1:35898)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.163952867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"ds_connection connected from Ok(127.0.0.1:40611)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.164093278Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"ds_connection connected from Ok(127.0.0.1:44588)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.164112616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.164732944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.164766912Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.164826528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165148996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165191473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[0]R flush_numbers[0..12]: [121, 131, 131, 131, 128, 129, 132, 129, 125, 132, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165267487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[0]R generation[0..12]: [73, 78, 78, 78, 76, 77, 79, 77, 74, 79, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165284296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165300835Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[1]R flush_numbers[0..12]: [121, 131, 131, 131, 128, 129, 132, 129, 125, 132, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165316674Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[1]R generation[0..12]: [73, 78, 78, 78, 76, 77, 79, 77, 74, 79, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165332033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165348342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[2]R flush_numbers[0..12]: [121, 131, 131, 131, 128, 129, 132, 129, 125, 132, 116, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165364071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[2]R generation[0..12]: [73, 78, 78, 78, 76, 77, 79, 77, 74, 79, 70, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.16537916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165395269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165408628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165427916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165441286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165454505Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"":"downstairs","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165470374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165483093Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"0","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165497112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"1","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165510921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"client":"2","":"downstairs","session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.16552749Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165541349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"b50a6796-8556-4d2f-b172-ecc1c7eef011 is now active with session: 0a725065-01b6-4573-9482-300f3dc0c463","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165556448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165569207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520,"session_id":"0a725065-01b6-4573-9482-300f3dc0c463"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165582606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} {"msg":"Activated sub_volume b50a6796-8556-4d2f-b172-ecc1c7eef011","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:49.165593845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 24576 data: 8 7 8 8 8 7 02/30 Read block 105 len 4096 03/30 Write block 130 len 36864 data: 5 5 6 7 8 8 12 9 11 04/30 Read block 483 len 4096 05/30 Read block 566 len 12288 06/30 Read block 39 len 40960 07/30 Flush 08/30 Flush 09/30 Write block 239 len 16384 data: 4 3 4 7 10/30 Write block 149 len 20480 data: 6 6 6 6 5 11/30 Write block 32 len 28672 data: 9 7 8 8 8 8 11 12/30 Write block 466 len 20480 data: 7 7 6 6 8 13/30 Write block 128 len 4096 data: 5 14/30 Write block 375 len 36864 data: 8 8 9 9 8 8 6 6 6 15/30 Flush 16/30 Write block 96 len 24576 data: 8 8 7 6 7 7 17/30 Write block 563 len 4096 data: 6 18/30 Read block 90 len 40960 19/30 Read block 107 len 20480 20/30 Write block 51 len 4096 data: 6 21/30 Read block 263 len 8192 22/30 Read block 345 len 36864 23/30 Read block 485 len 28672 24/30 Read block 308 len 4096 25/30 Write block 539 len 36864 data: 4 6 6 7 7 7 6 5 6 26/30 Write block 242 len 40960 data: 8 8 8 9 9 6 6 6 6 5 27/30 Read block 398 len 24576 28/30 Read block 22 len 4096 29/30 Write block 494 len 24576 data: 10 9 6 6 9 10 30/30 Write block 478 len 16384 data: 8 8 9 9 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:49.177130887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 80 78 80 135 131 135 F F F <--- 2 040-059 78 78 78 131 131 131 T F T <--- 3 060-079 78 78 78 131 131 131 F F F 4 080-099 76 76 76 128 128 128 T F T <--- 5 100-119 77 77 77 129 129 129 T F T <--- 6 120-139 80 79 80 135 132 135 F F F <--- 7 140-159 80 77 80 135 129 135 F F F <--- 8 160-179 74 74 74 125 125 125 F F F 9 180-199 79 79 79 132 132 132 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 80 74 80 135 123 135 F F F <--- 12 240-259 80 79 80 135 132 135 T F T <--- 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 78 78 78 130 130 130 F F F 16 320-339 79 79 79 132 132 132 F F F 17 340-359 77 77 77 129 129 129 F F F 18 360-379 80 66 80 135 107 135 F F F <--- 19 380-399 80 62 80 135 101 135 F F F <--- 20 400-419 80 68 80 133 113 133 F T F <--- 21 420-439 80 78 80 133 131 133 F T F <--- 22 440-459 76 76 76 128 128 128 F F F 23 460-479 80 78 80 135 131 135 T F T <--- 24 480-499 79 79 79 132 132 132 T F T <--- 25 500-519 78 78 78 130 130 130 F F F 26 520-539 78 78 78 131 131 131 T F T <--- 27 540-559 74 74 74 123 123 123 T F T <--- 28 560-579 77 77 77 129 129 129 T F T <--- 29 580-599 77 77 77 129 129 129 F F F Max gen: 80, Max flush: 135 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 121 121 121 F F F 1 020-039 80 80 80 135 135 135 F F F 2 040-059 81 81 81 136 136 136 F F F 3 060-079 78 78 78 131 131 131 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 81 81 81 136 136 136 F F F 6 120-139 80 80 80 135 135 135 F F F 7 140-159 80 80 80 135 135 135 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 79 79 79 132 132 132 F F F 10 200-219 70 70 70 116 116 116 F F F 11 220-239 80 80 80 135 135 135 F F F 12 240-259 81 81 81 136 136 136 F F F 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 78 78 78 130 130 130 F F F 16 320-339 79 79 79 132 132 132 F F F 17 340-359 77 77 77 129 129 129 F F F 18 360-379 80 80 80 135 135 135 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 80 80 80 133 133 133 F F F 21 420-439 80 80 80 133 133 133 F F F 22 440-459 76 76 76 128 128 128 F F F 23 460-479 81 81 81 136 136 136 F F F 24 480-499 81 81 81 136 136 136 F F F 25 500-519 78 78 78 130 130 130 F F F 26 520-539 81 81 81 136 136 136 F F F 27 540-559 81 81 81 136 136 136 F F F 28 560-579 81 81 81 136 136 136 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 81, Max flush: 136 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.511563445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.523264366Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.523587644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.523625631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.52363774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.523656189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"Crucible 9379a50f-4b20-4684-afac-12c98c4ace33 has session id: 91a8b368-c47f-4d84-bcb5-951fa6dd03a7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.523994646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Upstairs opts: Upstairs UUID: 9379a50f-4b20-4684-afac-12c98c4ace33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524027504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Crucible stats registered with UUID: 9379a50f-4b20-4684-afac-12c98c4ace33","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524042502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:53.524055622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524249518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524285776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524301145Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524395238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524412457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"9379a50f-4b20-4684-afac-12c98c4ace33 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524429886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524442695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524456674Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524470603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524484602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524498771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.52451567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52068)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524530869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50713)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524545688Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62216)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.524691508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525288797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525312776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525338054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525657112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525672571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[0]R flush_numbers[0..12]: [121, 135, 136, 131, 136, 136, 135, 135, 125, 132, 116, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525726847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[0]R generation[0..12]: [73, 80, 81, 78, 81, 81, 80, 80, 74, 79, 70, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525743536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525760095Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[1]R flush_numbers[0..12]: [121, 135, 136, 131, 136, 136, 135, 135, 125, 132, 116, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525775644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[1]R generation[0..12]: [73, 80, 81, 78, 81, 81, 80, 80, 74, 79, 70, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525790973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525809752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[2]R flush_numbers[0..12]: [121, 135, 136, 131, 136, 136, 135, 135, 125, 132, 116, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525825771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[2]R generation[0..12]: [73, 80, 81, 78, 81, 81, 80, 80, 74, 79, 70, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.52584086Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525857468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:34:53.525871088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525891056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525904475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525918164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"":"downstairs","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525932163Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525944753Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"0","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525958632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"client":"1","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525972231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1528,"client":"2","":"downstairs","session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.525996879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.526010338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"9379a50f-4b20-4684-afac-12c98c4ace33 is now active with session: 91a8b368-c47f-4d84-bcb5-951fa6dd03a7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.526025077Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.526039166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528,"session_id":"91a8b368-c47f-4d84-bcb5-951fa6dd03a7"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.526052585Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} {"msg":"Activated sub_volume 9379a50f-4b20-4684-afac-12c98c4ace33","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:53.526064254Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 16384 data: 9 10 11 11 02/30 Write block 324 len 36864 data: 7 6 6 7 6 4 4 5 6 03/30 Read block 5 len 20480 04/30 Write block 235 len 32768 data: 10 10 7 6 5 4 5 9 05/30 Write block 113 len 36864 data: 6 6 5 6 6 8 8 8 6 06/30 Read block 247 len 4096 07/30 Flush 08/30 Write block 60 len 32768 data: 7 8 9 10 12 11 10 10 09/30 Read block 487 len 28672 10/30 Read block 535 len 36864 11/30 Read block 386 len 16384 12/30 Write block 141 len 8192 data: 7 5 13/30 Write block 5 len 24576 data: 6 6 6 7 9 10 14/30 Read block 548 len 32768 15/30 Write block 340 len 12288 data: 4 6 6 16/30 Read block 182 len 24576 17/30 Read block 215 len 40960 18/30 Read block 180 len 4096 19/30 Write block 219 len 32768 data: 7 9 8 9 9 10 10 10 20/30 Write block 54 len 12288 data: 3 3 4 21/30 Write block 114 len 24576 data: 7 6 7 7 9 9 22/30 Read block 16 len 20480 23/30 Write block 533 len 36864 data: 6 6 4 4 4 4 5 7 7 24/30 Read block 112 len 32768 25/30 Flush 26/30 Write block 412 len 4096 data: 8 27/30 Write block 441 len 32768 data: 7 6 7 5 7 9 6 7 28/30 Write block 149 len 36864 data: 7 7 7 7 6 4 6 6 6 29/30 Write block 553 len 40960 data: 6 7 8 9 9 10 9 8 9 7 30/30 Write block 130 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 24 2 28 Done 28 6 28 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:53.538725219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 73 82 138 121 138 F F F <--- 1 020-039 80 80 80 135 135 135 F F F 2 040-059 82 81 82 138 136 138 F F F <--- 3 060-079 82 78 82 138 131 138 F F F <--- 4 080-099 81 81 81 136 136 136 F F F 5 100-119 82 81 82 138 136 138 F T F <--- 6 120-139 82 80 82 137 135 137 T T T <--- 7 140-159 82 80 82 138 135 138 T F T <--- 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 79 82 137 132 137 F T F <--- 10 200-219 82 70 82 138 116 138 F F F <--- 11 220-239 82 80 82 138 135 138 F T F <--- 12 240-259 82 81 82 137 136 137 F T F <--- 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 78 78 78 130 130 130 F F F 16 320-339 82 79 82 137 132 137 F T F <--- 17 340-359 82 77 82 138 129 138 F F F <--- 18 360-379 80 80 80 135 135 135 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 80 80 80 133 133 133 T F T <--- 21 420-439 80 80 80 133 133 133 F F F 22 440-459 76 76 76 128 128 128 T F T <--- 23 460-479 81 81 81 136 136 136 F F F 24 480-499 81 81 81 136 136 136 F F F 25 500-519 78 78 78 130 130 130 F F F 26 520-539 82 81 82 138 136 138 F F F <--- 27 540-559 82 81 82 138 136 138 T F T <--- 28 560-579 81 81 81 136 136 136 T F T <--- 29 580-599 77 77 77 129 129 129 F F F Max gen: 82, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 138 138 138 F F F 1 020-039 80 80 80 135 135 135 F F F 2 040-059 82 82 82 138 138 138 F F F 3 060-079 82 82 82 138 138 138 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 82 82 82 138 138 138 F F F 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 82 82 82 138 138 138 F F F 11 220-239 82 82 82 138 138 138 F F F 12 240-259 82 82 82 137 137 137 F F F 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 70 70 117 117 117 F F F 15 300-319 78 78 78 130 130 130 F F F 16 320-339 82 82 82 137 137 137 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 80 80 80 135 135 135 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 80 80 80 133 133 133 F F F 22 440-459 83 83 83 139 139 139 F F F 23 460-479 81 81 81 136 136 136 F F F 24 480-499 81 81 81 136 136 136 F F F 25 500-519 78 78 78 130 130 130 F F F 26 520-539 82 82 82 138 138 138 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 83 83 83 139 139 139 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 83, Max flush: 139 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.876436683Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:57.887219386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.887269543Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.89770812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898013029Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898050836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898063025Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898080744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"Crucible 819887ec-0b32-467a-908e-28e53d9718fe has session id: 13b27fc8-c7ea-4139-ad4d-2698fcb0a939","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898422321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Upstairs opts: Upstairs UUID: 819887ec-0b32-467a-908e-28e53d9718fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898456608Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Crucible stats registered with UUID: 819887ec-0b32-467a-908e-28e53d9718fe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898472267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:57.898488146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898647925Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898672964Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898687633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898812074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898833383Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"819887ec-0b32-467a-908e-28e53d9718fe active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898863921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.89887777Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898891889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898905808Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898919307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898934026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.898948065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"ds_connection connected from Ok(127.0.0.1:53867)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.89901196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"ds_connection connected from Ok(127.0.0.1:52190)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.899112694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"ds_connection connected from Ok(127.0.0.1:58768)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.899140472Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.899812486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.899851813Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900010692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.90019151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900222888Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[0]R flush_numbers[0..12]: [138, 135, 138, 138, 136, 138, 139, 139, 125, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900405045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[0]R generation[0..12]: [82, 80, 82, 82, 81, 82, 83, 83, 74, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900422134Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900439023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[1]R flush_numbers[0..12]: [138, 135, 138, 138, 136, 138, 139, 139, 125, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900454872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[1]R generation[0..12]: [82, 80, 82, 82, 81, 82, 83, 83, 74, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900470301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.90048638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[2]R flush_numbers[0..12]: [138, 135, 138, 138, 136, 138, 139, 139, 125, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900502149Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[2]R generation[0..12]: [82, 80, 82, 82, 81, 82, 83, 83, 74, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900517258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900533597Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Max found gen is 84","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2026-03-16T21:34:57.900555595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900569704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900583033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900596512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"":"downstairs","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900610661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.90062324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"0","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900637239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"1","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900653438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"client":"2","":"downstairs","session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900668017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900681376Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"819887ec-0b32-467a-908e-28e53d9718fe is now active with session: 13b27fc8-c7ea-4139-ad4d-2698fcb0a939","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900696355Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900709225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536,"session_id":"13b27fc8-c7ea-4139-ad4d-2698fcb0a939"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900722234Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} {"msg":"Activated sub_volume 819887ec-0b32-467a-908e-28e53d9718fe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:34:57.900733633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 296 len 4096 data: 6 02/30 Write block 318 len 8192 data: 7 6 03/30 Write block 111 len 4096 data: 5 04/30 Flush 05/30 Write block 436 len 8192 data: 8 11 06/30 Write block 564 len 32768 data: 6 4 5 5 4 2 5 6 07/30 Write block 513 len 28672 data: 5 7 8 9 10 11 7 08/30 Write block 369 len 28672 data: 6 7 7 6 6 6 9 09/30 Read block 405 len 4096 10/30 Read block 334 len 24576 11/30 Read block 532 len 20480 12/30 Read block 385 len 12288 13/30 Read block 59 len 36864 14/30 Read block 441 len 24576 15/30 Read block 592 len 24576 16/30 Read block 274 len 8192 17/30 Write block 234 len 12288 data: 10 11 11 18/30 Read block 457 len 4096 19/30 Write block 519 len 36864 data: 8 8 6 6 3 3 4 5 7 20/30 Read block 486 len 36864 21/30 Read block 364 len 12288 22/30 Flush 23/30 Write block 112 len 4096 data: 6 24/30 Read block 297 len 12288 25/30 Write block 212 len 12288 data: 5 5 5 26/30 Write block 103 len 40960 data: 7 5 5 6 4 5 4 4 6 7 27/30 Write block 318 len 20480 data: 8 7 7 7 7 28/30 Write block 61 len 28672 data: 9 10 11 13 12 11 11 29/30 Write block 257 len 12288 data: 5 5 6 30/30 Read block 495 len 4096 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 7 Sent Done Sent false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 3 30 Done 3 30 27 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:34:57.911281202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 138 138 138 F F F 1 020-039 80 80 80 135 135 135 F F F 2 040-059 82 82 82 138 138 138 F F F 3 060-079 82 82 82 138 138 138 F T T <--- 4 080-099 81 81 81 136 136 136 F F F 5 100-119 82 84 84 138 140 140 T T T <--- 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 82 82 82 138 138 138 F T T <--- 11 220-239 82 84 84 138 141 141 F F F <--- 12 240-259 82 82 82 137 137 137 F T T <--- 13 260-279 79 79 79 132 132 132 F F F 14 280-299 70 84 84 117 140 140 T F F <--- 15 300-319 78 84 84 130 140 140 T T T <--- 16 320-339 82 82 82 137 137 137 F T T <--- 17 340-359 82 82 82 138 138 138 F F F 18 360-379 80 84 84 135 141 141 F F F <--- 19 380-399 80 80 80 135 135 135 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 80 84 84 133 141 141 F F F <--- 22 440-459 83 83 83 139 139 139 F F F 23 460-479 81 81 81 136 136 136 F F F 24 480-499 81 81 81 136 136 136 F F F 25 500-519 78 84 84 130 141 141 F F F <--- 26 520-539 82 84 84 138 141 141 F F F <--- 27 540-559 83 83 83 139 139 139 F F F 28 560-579 83 84 84 139 141 141 F F F <--- 29 580-599 77 77 77 129 129 129 F F F Max gen: 84, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 138 138 138 F F F 1 020-039 80 80 80 135 135 135 F F F 2 040-059 82 82 82 138 138 138 F F F 3 060-079 85 85 85 142 142 142 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 85 85 85 142 142 142 F F F 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 84 84 84 141 141 141 F F F 12 240-259 85 85 85 142 142 142 F F F 13 260-279 79 79 79 132 132 132 F F F 14 280-299 84 84 84 140 140 140 F F F 15 300-319 85 85 85 142 142 142 F F F 16 320-339 85 85 85 142 142 142 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 84 84 84 141 141 141 F F F 19 380-399 80 80 80 135 135 135 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 83 83 83 139 139 139 F F F 23 460-479 81 81 81 136 136 136 F F F 24 480-499 81 81 81 136 136 136 F F F 25 500-519 84 84 84 141 141 141 F F F 26 520-539 84 84 84 141 141 141 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 85, Max flush: 142 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.266430535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.278301154Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.278620362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.278657629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.278669989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.278687427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"Crucible 0a24d109-e5ad-4704-a473-765605fa37fb has session id: 79d89524-25b1-4f1e-b463-a3b84d1e9a69","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279032814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Upstairs opts: Upstairs UUID: 0a24d109-e5ad-4704-a473-765605fa37fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279067222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Crucible stats registered with UUID: 0a24d109-e5ad-4704-a473-765605fa37fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.27908269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:02.27909626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279269708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279294866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279309405Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279441986Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279460475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"0a24d109-e5ad-4704-a473-765605fa37fb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279479973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279492662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279508741Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.27952298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279536949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279551368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279565677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:57855)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279614054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:50466)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279630143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"ds_connection connected from Ok(127.0.0.1:58632)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.279645002Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280377822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.2804069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280431218Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280751356Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280790124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[0]R flush_numbers[0..12]: [138, 135, 138, 142, 136, 142, 139, 139, 125, 137, 142, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280854809Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[0]R generation[0..12]: [82, 80, 82, 85, 81, 85, 83, 83, 74, 82, 85, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280871598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280890547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[1]R flush_numbers[0..12]: [138, 135, 138, 142, 136, 142, 139, 139, 125, 137, 142, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280906326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[1]R generation[0..12]: [82, 80, 82, 85, 81, 85, 83, 83, 74, 82, 85, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280921705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280937964Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[2]R flush_numbers[0..12]: [138, 135, 138, 142, 136, 142, 139, 139, 125, 137, 142, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280953303Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[2]R generation[0..12]: [82, 80, 82, 85, 81, 85, 83, 83, 74, 82, 85, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280968352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.280984381Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.28099791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281012069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281025448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281038747Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"":"downstairs","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281058705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281071165Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"0","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281085144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"1","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281099063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"client":"2","":"downstairs","session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281114912Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281128231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"0a24d109-e5ad-4704-a473-765605fa37fb is now active with session: 79d89524-25b1-4f1e-b463-a3b84d1e9a69","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.28114301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"Set Active after no reconciliation","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281158379Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544,"session_id":"79d89524-25b1-4f1e-b463-a3b84d1e9a69"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281172158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} {"msg":"Activated sub_volume 0a24d109-e5ad-4704-a473-765605fa37fb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:02.281183607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 482 len 20480 data: 10 8 5 6 8 02/30 Read block 386 len 36864 03/30 Read block 13 len 8192 04/30 Flush 05/30 Write block 24 len 16384 data: 9 5 3 5 06/30 Read block 588 len 36864 07/30 Write block 255 len 28672 data: 3 4 6 6 7 6 7 08/30 Flush 09/30 Write block 465 len 32768 data: 6 8 8 7 7 9 9 7 10/30 Flush 11/30 Write block 230 len 32768 data: 13 12 11 10 11 12 12 8 12/30 Flush 13/30 Write block 334 len 20480 data: 4 5 6 6 5 14/30 Read block 166 len 24576 15/30 Read block 299 len 12288 16/30 Read block 243 len 36864 17/30 Read block 280 len 12288 18/30 Read block 251 len 40960 19/30 Write block 248 len 16384 data: 7 7 7 6 20/30 Read block 434 len 12288 21/30 Flush 22/30 Write block 398 len 4096 data: 12 23/30 Read block 188 len 16384 24/30 Read block 380 len 28672 25/30 Read block 228 len 36864 26/30 Read block 271 len 32768 27/30 Read block 431 len 40960 28/30 Read block 513 len 12288 29/30 Write block 479 len 40960 data: 9 10 10 11 9 6 7 9 12 13 30/30 Write block 370 len 16384 data: 8 8 7 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 9 Done Done Sent false 1002 Acked Read 2 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:02.294172879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 138 138 138 F F F 1 020-039 86 86 80 144 144 135 F F F <--- 2 040-059 82 82 82 138 138 138 F F F 3 060-079 85 85 85 142 142 142 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 85 85 85 142 142 142 F F F 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 86 86 84 146 146 141 F F F <--- 12 240-259 86 86 85 147 147 142 F F F <--- 13 260-279 86 86 79 144 144 132 F F F <--- 14 280-299 84 84 84 140 140 140 F F F 15 300-319 85 85 85 142 142 142 F F F 16 320-339 86 86 85 147 147 142 F F F <--- 17 340-359 82 82 82 138 138 138 F F F 18 360-379 84 84 84 141 141 141 T T F <--- 19 380-399 80 80 80 135 135 135 T T F <--- 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 83 83 83 139 139 139 F F F 23 460-479 86 86 81 145 145 136 T T F <--- 24 480-499 86 86 81 143 143 136 T T F <--- 25 500-519 84 84 84 141 141 141 F F F 26 520-539 84 84 84 141 141 141 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 86, Max flush: 147 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 138 138 138 F F F 1 020-039 86 86 86 144 144 144 F F F 2 040-059 82 82 82 138 138 138 F F F 3 060-079 85 85 85 142 142 142 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 85 85 85 142 142 142 F F F 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 86 86 86 147 147 147 F F F 13 260-279 86 86 86 144 144 144 F F F 14 280-299 84 84 84 140 140 140 F F F 15 300-319 85 85 85 142 142 142 F F F 16 320-339 86 86 86 147 147 147 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 87 87 87 148 148 148 F F F 19 380-399 87 87 87 148 148 148 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 83 83 83 139 139 139 F F F 23 460-479 87 87 87 148 148 148 F F F 24 480-499 87 87 87 148 148 148 F F F 25 500-519 84 84 84 141 141 141 F F F 26 520-539 84 84 84 141 141 141 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 F F F 29 580-599 77 77 77 129 129 129 F F F Max gen: 87, Max flush: 148 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.606210938Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:06.616892638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.616942525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627172256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627466056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627503763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627515442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627533301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"Crucible 8d74cd83-2b6a-4122-ac56-5685d738d90a has session id: d16fc4aa-e356-49c3-a723-e1aba42119f2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627873118Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Upstairs opts: Upstairs UUID: 8d74cd83-2b6a-4122-ac56-5685d738d90a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627904716Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Crucible stats registered with UUID: 8d74cd83-2b6a-4122-ac56-5685d738d90a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.627922744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:06.627935854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628113161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62813853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628153479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62827394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628291659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"8d74cd83-2b6a-4122-ac56-5685d738d90a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628308618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628321417Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628335506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628349315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628362894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628377293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628391462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51488)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628406211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63523)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62842303Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44843)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.628437879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629078156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62915568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629285241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629426062Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629464239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[0]R flush_numbers[0..12]: [138, 144, 138, 142, 136, 142, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629646017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[0]R generation[0..12]: [82, 86, 82, 85, 81, 85, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629663036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629679694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[1]R flush_numbers[0..12]: [138, 144, 138, 142, 136, 142, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629695643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[1]R generation[0..12]: [82, 86, 82, 85, 81, 85, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629711272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629727831Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[2]R flush_numbers[0..12]: [138, 144, 138, 142, 136, 142, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62974366Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[2]R generation[0..12]: [82, 86, 82, 85, 81, 85, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629761709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629778308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629797446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629811775Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629825164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629838534Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"":"downstairs","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629852683Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629865512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"0","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629879531Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"1","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.62989575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"client":"2","":"downstairs","session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629910409Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629924088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"8d74cd83-2b6a-4122-ac56-5685d738d90a is now active with session: d16fc4aa-e356-49c3-a723-e1aba42119f2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629939477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629952126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552,"session_id":"d16fc4aa-e356-49c3-a723-e1aba42119f2"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629965645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} {"msg":"Activated sub_volume 8d74cd83-2b6a-4122-ac56-5685d738d90a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:06.629977114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 591 len 32768 data: 2 3 4 4 4 4 3 2 03/30 Flush 04/30 Write block 14 len 4096 data: 7 05/30 Read block 497 len 4096 06/30 Write block 456 len 20480 data: 11 9 8 9 9 07/30 Write block 316 len 28672 data: 6 7 9 8 8 8 8 08/30 Write block 110 len 16384 data: 5 7 8 7 09/30 Write block 68 len 8192 data: 11 10 10/30 Flush 11/30 Flush 12/30 Flush 13/30 Write block 283 len 20480 data: 6 7 6 5 5 14/30 Write block 335 len 4096 data: 6 15/30 Read block 140 len 32768 16/30 Write block 24 len 32768 data: 10 6 4 6 5 8 9 9 17/30 Read block 371 len 28672 18/30 Read block 197 len 24576 19/30 Read block 22 len 28672 20/30 Read block 342 len 36864 21/30 Write block 454 len 32768 data: 10 9 12 10 9 10 10 6 22/30 Write block 88 len 4096 data: 6 23/30 Write block 42 len 36864 data: 7 6 5 4 5 6 6 7 7 24/30 Read block 466 len 32768 25/30 Write block 471 len 16384 data: 10 8 6 5 26/30 Read block 14 len 20480 27/30 Read block 39 len 20480 28/30 Write block 11 len 20480 data: 8 10 8 8 7 29/30 Read block 269 len 8192 30/30 Read block 526 len 36864 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:06.641839714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 88 88 138 151 151 F T T <--- 1 020-039 86 86 86 144 144 144 F T T <--- 2 040-059 82 82 82 138 138 138 F T T <--- 3 060-079 85 88 88 142 151 151 F F F <--- 4 080-099 81 81 81 136 136 136 F T T <--- 5 100-119 85 88 88 142 151 151 F F F <--- 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 86 86 86 147 147 147 F F F 13 260-279 86 86 86 144 144 144 F F F 14 280-299 84 84 84 140 140 140 F T T <--- 15 300-319 85 88 88 142 151 151 F F F <--- 16 320-339 86 88 88 147 151 151 F T T <--- 17 340-359 82 82 82 138 138 138 F F F 18 360-379 87 87 87 148 148 148 F F F 19 380-399 87 87 87 148 148 148 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 83 88 88 139 151 151 F T T <--- 23 460-479 87 88 88 148 151 151 F T T <--- 24 480-499 87 87 87 148 148 148 F F F 25 500-519 84 84 84 141 141 141 F F F 26 520-539 84 84 84 141 141 141 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 F F F 29 580-599 77 88 88 129 150 150 F F F <--- Max gen: 88, Max flush: 151 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 152 152 152 F F F 1 020-039 89 89 89 152 152 152 F F F 2 040-059 89 89 89 152 152 152 F F F 3 060-079 88 88 88 151 151 151 F F F 4 080-099 89 89 89 152 152 152 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 83 83 83 139 139 139 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 F F F 10 200-219 85 85 85 142 142 142 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 86 86 86 147 147 147 F F F 13 260-279 86 86 86 144 144 144 F F F 14 280-299 89 89 89 152 152 152 F F F 15 300-319 88 88 88 151 151 151 F F F 16 320-339 89 89 89 152 152 152 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 87 87 87 148 148 148 F F F 19 380-399 87 87 87 148 148 148 F F F 20 400-419 83 83 83 139 139 139 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 89 89 89 152 152 152 F F F 23 460-479 89 89 89 152 152 152 F F F 24 480-499 87 87 87 148 148 148 F F F 25 500-519 84 84 84 141 141 141 F F F 26 520-539 84 84 84 141 141 141 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 F F F 29 580-599 88 88 88 150 150 150 F F F Max gen: 89, Max flush: 152 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.972711095Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.984549906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.984870304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.984907441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.984919421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.984937359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"Crucible e97e1486-d49e-46c6-a60f-240e0e7f1c40 has session id: c4f7031d-52f5-4677-9b1e-2506d1b21eb2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985237349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Upstairs opts: Upstairs UUID: e97e1486-d49e-46c6-a60f-240e0e7f1c40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985343052Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Crucible stats registered with UUID: e97e1486-d49e-46c6-a60f-240e0e7f1c40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.98536963Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:10.985383819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985530369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985556387Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985570806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985685748Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985707247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"e97e1486-d49e-46c6-a60f-240e0e7f1c40 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985739355Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985753274Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985767473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985781602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"2","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.98581183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985826419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985840528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60379)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985860486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33572)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985875635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"ds_connection connected from Ok(127.0.0.1:35224)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.985890484Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.98668105Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.986709208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.986726097Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"2","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987068374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987111501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[0]R flush_numbers[0..12]: [152, 152, 152, 151, 152, 151, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987177056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[0]R generation[0..12]: [89, 89, 89, 88, 89, 88, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987194005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987210304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[1]R flush_numbers[0..12]: [152, 152, 152, 151, 152, 151, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987226103Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[1]R generation[0..12]: [89, 89, 89, 88, 89, 88, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987241152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987257401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[2]R flush_numbers[0..12]: [152, 152, 152, 151, 152, 151, 139, 139, 125, 137, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.98727317Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[2]R generation[0..12]: [89, 89, 89, 88, 89, 88, 83, 83, 74, 82, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987288329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987304458Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:35:10.987317837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987335606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987348935Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987362304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"":"downstairs","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987376443Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987391202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"0","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987405441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"1","":"downstairs","session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.98741937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"client":"2","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987436769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987449918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"e97e1486-d49e-46c6-a60f-240e0e7f1c40 is now active with session: c4f7031d-52f5-4677-9b1e-2506d1b21eb2","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987464587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987477466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560,"session_id":"c4f7031d-52f5-4677-9b1e-2506d1b21eb2"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987490915Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} {"msg":"Activated sub_volume e97e1486-d49e-46c6-a60f-240e0e7f1c40","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:10.987502344Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 12288 data: 9 9 7 02/30 Flush 03/30 Write block 201 len 8192 data: 10 10 04/30 Write block 494 len 36864 data: 11 10 7 7 10 11 10 11 13 05/30 Flush 06/30 Flush 07/30 Write block 268 len 36864 data: 4 5 6 7 8 7 7 7 6 08/30 Write block 240 len 40960 data: 5 6 10 9 9 10 10 7 8 8 09/30 Read block 163 len 20480 10/30 Write block 366 len 8192 data: 9 9 11/30 Read block 32 len 8192 12/30 Write block 122 len 24576 data: 5 5 6 4 5 6 13/30 Read block 476 len 8192 14/30 Write block 383 len 28672 data: 7 7 7 8 9 10 11 15/30 Write block 414 len 4096 data: 8 16/30 Write block 366 len 32768 data: 10 10 7 7 9 9 8 8 17/30 Read block 454 len 40960 18/30 Read block 460 len 8192 19/30 Read block 466 len 24576 20/30 Flush 21/30 Write block 595 len 4096 data: 5 22/30 Read block 128 len 28672 23/30 Read block 300 len 24576 24/30 Write block 38 len 32768 data: 12 13 11 8 8 7 6 5 25/30 Flush 26/30 Write block 457 len 36864 data: 11 10 11 11 7 6 7 7 7 27/30 Write block 564 len 4096 data: 7 28/30 Write block 336 len 8192 data: 7 7 29/30 Write block 190 len 12288 data: 6 6 7 30/30 Write block 389 len 32768 data: 12 11 11 11 13 10 12 12 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 9 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 21 27 Done 21 21 3 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:10.998900845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 152 152 152 F F F 1 020-039 90 90 89 157 157 152 F F F <--- 2 040-059 90 90 89 157 157 152 F F F <--- 3 060-079 88 88 88 151 151 151 F F F 4 080-099 89 89 89 152 152 152 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 90 90 83 156 156 139 F F F <--- 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 82 82 82 137 137 137 T T F <--- 10 200-219 90 90 90 154 154 154 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 90 90 86 156 156 147 F F T <--- 13 260-279 90 90 86 156 156 144 F F T <--- 14 280-299 89 89 89 152 152 152 F F F 15 300-319 88 88 88 151 151 151 F F F 16 320-339 89 89 89 152 152 152 T T F <--- 17 340-359 82 82 82 138 138 138 F F F 18 360-379 90 90 87 156 156 148 F F F <--- 19 380-399 90 90 87 156 156 148 T T F <--- 20 400-419 90 90 83 156 156 139 F F F <--- 21 420-439 84 84 84 141 141 141 F F F 22 440-459 89 89 89 152 152 152 T T F <--- 23 460-479 89 89 89 152 152 152 T T F <--- 24 480-499 90 90 90 154 154 154 F F F 25 500-519 90 90 90 154 154 154 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 84 84 84 141 141 141 T T F <--- 29 580-599 90 90 88 157 157 150 F F F <--- Max gen: 90, Max flush: 157 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 152 152 152 F F F 1 020-039 90 90 90 157 157 157 F F F 2 040-059 90 90 90 157 157 157 F F F 3 060-079 88 88 88 151 151 151 F F F 4 080-099 89 89 89 152 152 152 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 74 74 125 125 125 F F F 9 180-199 91 91 91 158 158 158 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 90 90 90 156 156 156 F F F 13 260-279 90 90 90 156 156 156 F F F 14 280-299 89 89 89 152 152 152 F F F 15 300-319 88 88 88 151 151 151 F F F 16 320-339 91 91 91 158 158 158 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 90 90 90 156 156 156 F F F 19 380-399 91 91 91 158 158 158 F F F 20 400-419 90 90 90 156 156 156 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 91 91 91 158 158 158 F F F 23 460-479 91 91 91 158 158 158 F F F 24 480-499 90 90 90 154 154 154 F F F 25 500-519 90 90 90 154 154 154 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 83 83 83 139 139 139 F F F 28 560-579 91 91 91 158 158 158 F F F 29 580-599 90 90 90 157 157 157 F F F Max gen: 91, Max flush: 158 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.326606413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:15.337307252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.337360858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.347654785Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.347952535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.347990122Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348002801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.34802083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"Crucible d7bc912e-08a9-4b30-add4-3e9836948dbc has session id: 67ae74ce-fac9-41f1-adbe-21d28da78d42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348372916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Upstairs opts: Upstairs UUID: d7bc912e-08a9-4b30-add4-3e9836948dbc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348407434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Crucible stats registered with UUID: d7bc912e-08a9-4b30-add4-3e9836948dbc","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348422743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:15.348436342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348617769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348653467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348668866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.3487578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348775858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"d7bc912e-08a9-4b30-add4-3e9836948dbc active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348795877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348808776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348822635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348836334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348850153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348864582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348881281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:54812)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.34889662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:39069)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348911649Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:41679)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.348926618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.349676377Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.349704205Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.349850585Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350053961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.35007066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[0]R flush_numbers[0..12]: [152, 157, 157, 151, 152, 151, 156, 139, 125, 158, 154, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350275806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[0]R generation[0..12]: [89, 90, 90, 88, 89, 88, 90, 83, 74, 91, 90, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350294615Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350311203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[1]R flush_numbers[0..12]: [152, 157, 157, 151, 152, 151, 156, 139, 125, 158, 154, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350334622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[1]R generation[0..12]: [89, 90, 90, 88, 89, 88, 90, 83, 74, 91, 90, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350350091Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.35036624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[2]R flush_numbers[0..12]: [152, 157, 157, 151, 152, 151, 156, 139, 125, 158, 154, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350385448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[2]R generation[0..12]: [89, 90, 90, 88, 89, 88, 90, 83, 74, 91, 90, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350400927Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350417016Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:35:15.350430565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350448204Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350461563Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350475282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"":"downstairs","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350489231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.35050172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"0","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.3505159Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"1","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350529789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"client":"2","":"downstairs","session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"All required reconciliation work is completed","v":0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350546817Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350560606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"d7bc912e-08a9-4b30-add4-3e9836948dbc is now active with session: 67ae74ce-fac9-41f1-adbe-21d28da78d42","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350575955Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350590434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568,"session_id":"67ae74ce-fac9-41f1-adbe-21d28da78d42"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350604003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} {"msg":"Activated sub_volume d7bc912e-08a9-4b30-add4-3e9836948dbc","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:15.350615373Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 36864 data: 9 7 9 9 8 9 7 8 9 02/30 Read block 563 len 36864 03/30 Read block 133 len 28672 04/30 Write block 297 len 8192 data: 8 8 05/30 Write block 99 len 28672 data: 7 8 8 8 8 6 6 06/30 Read block 354 len 40960 07/30 Flush 08/30 Write block 223 len 20480 data: 10 11 11 11 13 09/30 Read block 173 len 16384 10/30 Write block 541 len 12288 data: 8 8 8 11/30 Write block 95 len 32768 data: 6 9 9 8 8 9 9 9 12/30 Read block 282 len 4096 13/30 Flush 14/30 Write block 564 len 12288 data: 8 5 6 15/30 Write block 170 len 4096 data: 8 16/30 Read block 192 len 12288 17/30 Read block 467 len 32768 18/30 Write block 2 len 36864 data: 5 6 5 7 7 7 8 10 11 19/30 Flush 20/30 Write block 457 len 12288 data: 12 11 12 21/30 Write block 248 len 4096 data: 9 22/30 Write block 180 len 20480 data: 8 9 10 8 7 23/30 Read block 302 len 40960 24/30 Read block 310 len 16384 25/30 Write block 363 len 8192 data: 7 7 26/30 Write block 257 len 8192 data: 7 7 27/30 Write block 237 len 12288 data: 9 7 6 28/30 Read block 342 len 36864 29/30 Write block 264 len 32768 data: 7 7 5 3 5 6 7 8 30/30 Write block 378 len 40960 data: 10 9 9 7 7 8 8 8 9 10 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 4 28 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:15.362689108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 92 92 152 161 161 F F F <--- 1 020-039 90 90 90 157 157 157 F F F 2 040-059 90 90 90 157 157 157 F F F 3 060-079 88 88 88 151 151 151 F F F 4 080-099 89 92 92 152 160 160 F F F <--- 5 100-119 88 92 92 151 160 160 F F F <--- 6 120-139 90 90 90 156 156 156 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 74 92 92 125 161 161 F F F <--- 9 180-199 91 91 91 158 158 158 F T T <--- 10 200-219 90 90 90 154 154 154 F F F 11 220-239 86 92 92 146 160 160 F T T <--- 12 240-259 90 90 90 156 156 156 F T T <--- 13 260-279 90 90 90 156 156 156 F T T <--- 14 280-299 89 92 92 152 159 159 T F F <--- 15 300-319 88 88 88 151 151 151 F F F 16 320-339 91 91 91 158 158 158 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 90 90 90 156 156 156 F T T <--- 19 380-399 91 91 91 158 158 158 F T T <--- 20 400-419 90 92 92 156 159 159 T F F <--- 21 420-439 84 84 84 141 141 141 F F F 22 440-459 91 91 91 158 158 158 F T T <--- 23 460-479 91 91 91 158 158 158 F F F 24 480-499 90 90 90 154 154 154 F F F 25 500-519 90 90 90 154 154 154 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 83 92 92 139 160 160 F F F <--- 28 560-579 91 92 92 158 161 161 F F F <--- 29 580-599 90 90 90 157 157 157 F F F Max gen: 92, Max flush: 161 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 161 161 161 F F F 1 020-039 90 90 90 157 157 157 F F F 2 040-059 90 90 90 157 157 157 F F F 3 060-079 88 88 88 151 151 151 F F F 4 080-099 92 92 92 160 160 160 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 83 83 83 139 139 139 F F F 8 160-179 92 92 92 161 161 161 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 93 93 93 162 162 162 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 93 93 93 162 162 162 F F F 14 280-299 92 92 92 159 159 159 F F F 15 300-319 88 88 88 151 151 151 F F F 16 320-339 91 91 91 158 158 158 F F F 17 340-359 82 82 82 138 138 138 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 92 92 92 159 159 159 F F F 21 420-439 84 84 84 141 141 141 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 91 91 91 158 158 158 F F F 24 480-499 90 90 90 154 154 154 F F F 25 500-519 90 90 90 154 154 154 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 92 92 92 160 160 160 F F F 28 560-579 92 92 92 161 161 161 F F F 29 580-599 90 90 90 157 157 157 F F F Max gen: 93, Max flush: 162 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.702757471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.714607361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.71492321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.714960437Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.714972846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.714990655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"Crucible f463d8d2-ab91-41e5-ba38-2318915621fe has session id: 88b8585b-0dc1-4e6b-9bd7-78e1f321841b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715319243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Upstairs opts: Upstairs UUID: f463d8d2-ab91-41e5-ba38-2318915621fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.71535397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Crucible stats registered with UUID: f463d8d2-ab91-41e5-ba38-2318915621fe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715369709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:19.715383308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715559106Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715583705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715602143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715727195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715745294Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"f463d8d2-ab91-41e5-ba38-2318915621fe active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715767712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715780571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.71579471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715808599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715822508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715837077Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715851436Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54174)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715886384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61131)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715902693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40511)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.715917622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.716724647Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.716753705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.716778803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717103571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.71711837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[0]R flush_numbers[0..12]: [161, 157, 157, 151, 160, 160, 156, 139, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717185535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[0]R generation[0..12]: [92, 90, 90, 88, 92, 92, 90, 83, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717202404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717218793Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[1]R flush_numbers[0..12]: [161, 157, 157, 151, 160, 160, 156, 139, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717234312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[1]R generation[0..12]: [92, 90, 90, 88, 92, 92, 90, 83, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717249391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.71726536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[2]R flush_numbers[0..12]: [161, 157, 157, 151, 160, 160, 156, 139, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717281249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[2]R generation[0..12]: [92, 90, 90, 88, 92, 92, 90, 83, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717296578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717312846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717326365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717346154Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717359253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717374362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"":"downstairs","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717388761Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.7174017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"0","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717415789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"1","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717430198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"client":"2","":"downstairs","session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717444617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717460276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"f463d8d2-ab91-41e5-ba38-2318915621fe is now active with session: 88b8585b-0dc1-4e6b-9bd7-78e1f321841b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717475385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717488094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576,"session_id":"88b8585b-0dc1-4e6b-9bd7-78e1f321841b"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717501384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} {"msg":"Activated sub_volume f463d8d2-ab91-41e5-ba38-2318915621fe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:19.717512703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 16384 data: 6 7 7 6 02/30 Write block 394 len 28672 data: 11 13 13 12 13 11 10 03/30 Write block 578 len 36864 data: 5 6 4 6 6 7 8 7 6 04/30 Flush 05/30 Write block 421 len 40960 data: 9 8 8 9 9 12 13 15 16 14 06/30 Write block 266 len 16384 data: 6 4 6 7 07/30 Write block 503 len 12288 data: 17 14 13 08/30 Read block 32 len 32768 09/30 Read block 260 len 36864 10/30 Write block 560 len 28672 data: 9 10 8 7 9 6 7 11/30 Write block 355 len 4096 data: 11 12/30 Read block 68 len 28672 13/30 Write block 486 len 12288 data: 10 13 14 14/30 Write block 483 len 24576 data: 10 7 8 11 14 15 15/30 Write block 528 len 12288 data: 8 8 8 16/30 Read block 444 len 28672 17/30 Write block 66 len 12288 data: 12 12 12 18/30 Write block 142 len 4096 data: 6 19/30 Write block 306 len 24576 data: 6 6 6 6 6 6 20/30 Read block 421 len 40960 21/30 Write block 533 len 16384 data: 7 7 5 5 22/30 Read block 277 len 24576 23/30 Read block 370 len 40960 24/30 Read block 114 len 20480 25/30 Read block 576 len 8192 26/30 Flush 27/30 Write block 284 len 20480 data: 8 7 6 6 7 28/30 Read block 87 len 40960 29/30 Write block 55 len 24576 data: 4 5 5 5 5 8 30/30 Read block 263 len 24576 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 1 26 Done 26 1 25 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1003)) Some(JobId(1025)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:19.729374152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 161 161 161 F F F 1 020-039 90 90 90 157 157 157 F F F 2 040-059 90 90 90 157 157 157 T F T <--- 3 060-079 94 88 94 164 151 164 T F T <--- 4 080-099 92 92 92 160 160 160 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 94 83 94 164 139 164 F F F <--- 8 160-179 92 92 92 161 161 161 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 93 93 93 162 162 162 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 93 94 164 162 164 F F F <--- 14 280-299 92 92 92 159 159 159 T F T <--- 15 300-319 94 88 94 164 151 164 F F F <--- 16 320-339 91 91 91 158 158 158 F F F 17 340-359 94 82 94 164 138 164 F F F <--- 18 360-379 93 93 93 162 162 162 F F F 19 380-399 94 94 94 163 163 163 F F F 20 400-419 94 94 94 163 163 163 F F F 21 420-439 94 84 94 164 141 164 F T F <--- 22 440-459 93 93 93 162 162 162 F F F 23 460-479 91 91 91 158 158 158 F F F 24 480-499 94 90 94 164 154 164 F F F <--- 25 500-519 94 90 94 164 154 164 F F F <--- 26 520-539 94 90 94 164 153 164 F F F <--- 27 540-559 94 94 94 163 163 163 F F F 28 560-579 94 94 94 164 163 164 F F F <--- 29 580-599 94 94 94 163 163 163 F F F Max gen: 94, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 161 161 161 F F F 1 020-039 90 90 90 157 157 157 F F F 2 040-059 95 95 95 165 165 165 F F F 3 060-079 95 95 95 165 165 165 F F F 4 080-099 92 92 92 160 160 160 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 94 94 94 164 164 164 F F F 8 160-179 92 92 92 161 161 161 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 93 93 93 162 162 162 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 95 95 95 165 165 165 F F F 15 300-319 94 94 94 164 164 164 F F F 16 320-339 91 91 91 158 158 158 F F F 17 340-359 94 94 94 164 164 164 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 94 94 94 163 163 163 F F F 20 400-419 94 94 94 163 163 163 F F F 21 420-439 94 94 94 164 164 164 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 91 91 91 158 158 158 F F F 24 480-499 94 94 94 164 164 164 F F F 25 500-519 94 94 94 164 164 164 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 94 94 94 163 163 163 F F F 28 560-579 94 94 94 164 164 164 F F F 29 580-599 94 94 94 163 163 163 F F F Max gen: 95, Max flush: 165 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.049803917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.061536925Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.061861643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06189927Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06191177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.061932138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"Crucible c5ee61c3-1024-481d-971a-1a6e786db251 has session id: 75ea9dfe-597f-4161-948e-86c776d4f22b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062286684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Upstairs opts: Upstairs UUID: c5ee61c3-1024-481d-971a-1a6e786db251, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062320992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Crucible stats registered with UUID: c5ee61c3-1024-481d-971a-1a6e786db251","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062336931Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:24.06235015Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062500579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062536977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062552946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062663358Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062681197Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"c5ee61c3-1024-481d-971a-1a6e786db251 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062696946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062709765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062723774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062737843Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062751882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062768811Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06278352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57965)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062833417Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57605)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062861535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33852)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.062877874Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.063607674Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.063636672Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.063678949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.063972819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064018626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[0]R flush_numbers[0..12]: [161, 157, 165, 165, 160, 160, 156, 164, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064058143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[0]R generation[0..12]: [92, 90, 95, 95, 92, 92, 90, 94, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064075032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064091711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[1]R flush_numbers[0..12]: [161, 157, 165, 165, 160, 160, 156, 164, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06410751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[1]R generation[0..12]: [92, 90, 95, 95, 92, 92, 90, 94, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064123259Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064142757Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[2]R flush_numbers[0..12]: [161, 157, 165, 165, 160, 160, 156, 164, 161, 162, 154, 162]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064158756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[2]R generation[0..12]: [92, 90, 95, 95, 92, 92, 90, 94, 92, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064174305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064190154Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064209893Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064224112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064237641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06425119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"":"downstairs","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064265819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064278538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"0","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064292807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"1","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-16T21:35:24.064306746Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"client":"2","":"downstairs","session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064324185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064337774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"c5ee61c3-1024-481d-971a-1a6e786db251 is now active with session: 75ea9dfe-597f-4161-948e-86c776d4f22b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064352573Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064367172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584,"session_id":"75ea9dfe-597f-4161-948e-86c776d4f22b"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.064380481Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} {"msg":"Activated sub_volume c5ee61c3-1024-481d-971a-1a6e786db251","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:24.06439209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 32768 data: 14 18 15 14 13 14 11 10 02/30 Write block 383 len 16384 data: 9 9 9 10 03/30 Read block 118 len 8192 04/30 Write block 25 len 24576 data: 7 5 7 6 9 10 05/30 Write block 382 len 24576 data: 8 10 10 10 11 11 06/30 Write block 585 len 36864 data: 8 7 4 3 3 3 3 4 5 07/30 Read block 17 len 20480 08/30 Flush 09/30 Write block 84 len 16384 data: 5 4 3 3 10/30 Read block 73 len 8192 11/30 Read block 287 len 8192 12/30 Read block 296 len 16384 13/30 Write block 503 len 36864 data: 19 16 15 14 15 12 11 10 9 14/30 Read block 360 len 12288 15/30 Write block 175 len 16384 data: 9 8 7 6 16/30 Write block 494 len 4096 data: 12 17/30 Read block 179 len 20480 18/30 Read block 15 len 16384 19/30 Write block 9 len 16384 data: 11 12 9 11 20/30 Read block 155 len 16384 21/30 Write block 451 len 16384 data: 11 13 11 11 22/30 Read block 135 len 12288 23/30 Write block 285 len 20480 data: 8 7 7 8 6 24/30 Read block 285 len 12288 25/30 Write block 170 len 4096 data: 9 26/30 Read block 535 len 12288 27/30 Read block 346 len 24576 28/30 Read block 252 len 16384 29/30 Write block 222 len 12288 data: 10 11 12 30/30 Write block 300 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:24.075322743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 161 161 161 T T F <--- 1 020-039 96 96 90 166 166 157 F F F <--- 2 040-059 95 95 95 165 165 165 F F F 3 060-079 95 95 95 165 165 165 F F F 4 080-099 92 92 92 160 160 160 T T F <--- 5 100-119 92 92 92 160 160 160 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 94 94 94 164 164 164 F F F 8 160-179 92 92 92 161 161 161 T T F <--- 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 93 93 93 162 162 162 T T F <--- 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 95 95 95 165 165 165 T T F <--- 15 300-319 94 94 94 164 164 164 T T F <--- 16 320-339 91 91 91 158 158 158 F F F 17 340-359 94 94 94 164 164 164 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 96 96 94 166 166 163 F F F <--- 20 400-419 94 94 94 163 163 163 F F F 21 420-439 94 94 94 164 164 164 F F F 22 440-459 93 93 93 162 162 162 T T F <--- 23 460-479 91 91 91 158 158 158 F F F 24 480-499 94 94 94 164 164 164 T T F <--- 25 500-519 96 96 94 166 166 164 T T T <--- 26 520-539 94 94 94 164 164 164 F F F 27 540-559 94 94 94 163 163 163 F F F 28 560-579 94 94 94 164 164 164 F F F 29 580-599 96 96 94 166 166 163 F F F <--- Max gen: 96, Max flush: 166 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 167 167 167 F F F 1 020-039 96 96 96 166 166 166 F F F 2 040-059 95 95 95 165 165 165 F F F 3 060-079 95 95 95 165 165 165 F F F 4 080-099 97 97 97 167 167 167 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 90 90 90 156 156 156 F F F 7 140-159 94 94 94 164 164 164 F F F 8 160-179 97 97 97 167 167 167 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 97 97 97 167 167 167 F F F 15 300-319 97 97 97 167 167 167 F F F 16 320-339 91 91 91 158 158 158 F F F 17 340-359 94 94 94 164 164 164 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 96 96 96 166 166 166 F F F 20 400-419 94 94 94 163 163 163 F F F 21 420-439 94 94 94 164 164 164 F F F 22 440-459 97 97 97 167 167 167 F F F 23 460-479 91 91 91 158 158 158 F F F 24 480-499 97 97 97 167 167 167 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 94 94 94 163 163 163 F F F 28 560-579 94 94 94 164 164 164 F F F 29 580-599 96 96 96 166 166 166 F F F Max gen: 97, Max flush: 167 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.398887464Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.41065845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411014386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411054553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411067322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411085611Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"Crucible e4a1a7b3-5271-48b4-a446-7473f1a7d412 has session id: 6d8204b1-e901-49f2-8165-7ec35a1cecf0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411431077Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Upstairs opts: Upstairs UUID: e4a1a7b3-5271-48b4-a446-7473f1a7d412, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411465705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Crucible stats registered with UUID: e4a1a7b3-5271-48b4-a446-7473f1a7d412","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411481694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:28.411494793Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411671711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411700019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411715188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411837959Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411855648Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"e4a1a7b3-5271-48b4-a446-7473f1a7d412 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411872397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411885046Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411898995Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411912764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411960851Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.411985699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412000668Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53969)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412015787Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38663)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412030526Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50353)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412143598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412659183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.412806483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.41285075Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413029708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.41315028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[0]R flush_numbers[0..12]: [167, 166, 165, 165, 167, 160, 156, 164, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413224605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[0]R generation[0..12]: [97, 96, 95, 95, 97, 92, 90, 94, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413241173Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413257812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[1]R flush_numbers[0..12]: [167, 166, 165, 165, 167, 160, 156, 164, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413273591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[1]R generation[0..12]: [97, 96, 95, 95, 97, 92, 90, 94, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.4132886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413319168Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[2]R flush_numbers[0..12]: [167, 166, 165, 165, 167, 160, 156, 164, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413334557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[2]R generation[0..12]: [97, 96, 95, 95, 97, 92, 90, 94, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413349846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413366465Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413385884Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413399913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413415852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413429611Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"":"downstairs","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.4134433Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413455989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"0","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413470428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"1","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-03-16T21:35:28.413487547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"client":"2","":"downstairs","session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413502556Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413516335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"e4a1a7b3-5271-48b4-a446-7473f1a7d412 is now active with session: 6d8204b1-e901-49f2-8165-7ec35a1cecf0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413531514Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413544153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592,"session_id":"6d8204b1-e901-49f2-8165-7ec35a1cecf0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413557172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} {"msg":"Activated sub_volume e4a1a7b3-5271-48b4-a446-7473f1a7d412","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:28.413568421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 313 len 24576 data: 7 8 8 7 8 10 02/30 Write block 46 len 12288 data: 6 7 7 03/30 Read block 86 len 28672 04/30 Write block 447 len 12288 data: 7 8 6 05/30 Read block 262 len 36864 06/30 Read block 466 len 36864 07/30 Read block 436 len 12288 08/30 Write block 406 len 24576 data: 10 11 9 10 8 10 09/30 Read block 261 len 20480 10/30 Read block 119 len 36864 11/30 Read block 144 len 36864 12/30 Read block 157 len 4096 13/30 Write block 329 len 8192 data: 5 5 14/30 Write block 120 len 36864 data: 9 7 6 6 7 5 6 7 6 15/30 Read block 545 len 36864 16/30 Read block 528 len 4096 17/30 Read block 579 len 4096 18/30 Read block 399 len 28672 19/30 Flush 20/30 Write block 15 len 8192 data: 8 7 21/30 Write block 139 len 40960 data: 8 8 8 7 5 7 6 4 4 5 22/30 Write block 384 len 20480 data: 11 11 12 12 11 23/30 Read block 91 len 24576 24/30 Read block 5 len 20480 25/30 Write block 547 len 12288 data: 8 8 7 26/30 Read block 257 len 12288 27/30 Write block 452 len 24576 data: 14 12 12 10 13 13 28/30 Write block 472 len 40960 data: 9 7 6 5 4 5 9 10 11 11 29/30 Read block 84 len 36864 30/30 Write block 63 len 40960 data: 12 14 13 13 13 13 11 8 8 7 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:28.426301771Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 167 167 167 T F T <--- 1 020-039 96 96 96 166 166 166 F F F 2 040-059 98 95 98 168 165 168 F F F <--- 3 060-079 95 95 95 165 165 165 T F T <--- 4 080-099 97 97 97 167 167 167 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 98 90 98 168 156 168 T F T <--- 7 140-159 94 94 94 164 164 164 T F T <--- 8 160-179 97 97 97 167 167 167 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 97 97 97 167 167 167 F F F 15 300-319 98 97 98 168 167 168 F F F <--- 16 320-339 98 91 98 168 158 168 F F F <--- 17 340-359 94 94 94 164 164 164 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 96 96 96 166 166 166 T F T <--- 20 400-419 98 94 98 168 163 168 F F F <--- 21 420-439 94 94 94 164 164 164 F F F 22 440-459 98 97 98 168 167 168 T F T <--- 23 460-479 91 91 91 158 158 158 T F T <--- 24 480-499 97 97 97 167 167 167 T F T <--- 25 500-519 97 97 97 167 167 167 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 94 94 94 163 163 163 T F T <--- 28 560-579 94 94 94 164 164 164 F F F 29 580-599 96 96 96 166 166 166 F F F Max gen: 98, Max flush: 168 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 169 169 169 F F F 1 020-039 96 96 96 166 166 166 F F F 2 040-059 98 98 98 168 168 168 F F F 3 060-079 99 99 99 169 169 169 F F F 4 080-099 97 97 97 167 167 167 F F F 5 100-119 92 92 92 160 160 160 F F F 6 120-139 99 99 99 169 169 169 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 97 97 97 167 167 167 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 97 97 97 167 167 167 F F F 15 300-319 98 98 98 168 168 168 F F F 16 320-339 98 98 98 168 168 168 F F F 17 340-359 94 94 94 164 164 164 F F F 18 360-379 93 93 93 162 162 162 F F F 19 380-399 99 99 99 169 169 169 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 94 94 94 164 164 164 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 94 94 94 164 164 164 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 94 94 94 164 164 164 F F F 29 580-599 96 96 96 166 166 166 F F F Max gen: 99, Max flush: 169 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.752705118Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.764412798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.764739766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.764777813Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.764793102Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.764811091Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"Crucible 59ff1370-08f2-46eb-b3ad-129e878b9956 has session id: 37fa8565-98c8-4c3d-aa29-262a914f97f9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765168586Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Upstairs opts: Upstairs UUID: 59ff1370-08f2-46eb-b3ad-129e878b9956, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765199444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Crucible stats registered with UUID: 59ff1370-08f2-46eb-b3ad-129e878b9956","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765214203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:32.765227612Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.76540858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765433508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765448157Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765570619Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765588138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"59ff1370-08f2-46eb-b3ad-129e878b9956 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765611136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765623865Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765637984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765652423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765668072Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765682791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.76569701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35943)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765734898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57835)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765750957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48078)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.765766006Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766517474Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766545602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766562561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766897018Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766912037Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[0]R flush_numbers[0..12]: [169, 166, 168, 169, 167, 160, 169, 169, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766946705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[0]R generation[0..12]: [99, 96, 98, 99, 97, 92, 99, 99, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766963524Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766980063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[1]R flush_numbers[0..12]: [169, 166, 168, 169, 167, 160, 169, 169, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.766999061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[1]R generation[0..12]: [99, 96, 98, 99, 97, 92, 99, 99, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.7670148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767031679Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[2]R flush_numbers[0..12]: [169, 166, 168, 169, 167, 160, 169, 169, 167, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767047308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[2]R generation[0..12]: [99, 96, 98, 99, 97, 92, 99, 99, 97, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767062547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767078646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767091925Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767111394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767124973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767138202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"":"downstairs","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767152171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.76716493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"0","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767178949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"1","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767193088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"client":"2","":"downstairs","session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:35:32.767207677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767224976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"59ff1370-08f2-46eb-b3ad-129e878b9956 is now active with session: 37fa8565-98c8-4c3d-aa29-262a914f97f9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767240045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767252744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600,"session_id":"37fa8565-98c8-4c3d-aa29-262a914f97f9"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767266333Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} {"msg":"Activated sub_volume 59ff1370-08f2-46eb-b3ad-129e878b9956","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:32.767277692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 374 len 28672 data: 7 10 9 10 11 10 10 03/30 Write block 589 len 4096 data: 4 04/30 Write block 389 len 8192 data: 13 12 05/30 Write block 336 len 40960 data: 8 8 6 6 5 7 7 6 6 5 06/30 Write block 164 len 28672 data: 10 13 11 10 10 9 10 07/30 Read block 177 len 40960 08/30 Read block 350 len 4096 09/30 Write block 28 len 24576 data: 7 10 11 10 10 8 10/30 Read block 324 len 8192 11/30 Write block 528 len 28672 data: 9 9 9 8 7 8 8 12/30 Flush 13/30 Write block 578 len 8192 data: 6 7 14/30 Read block 279 len 4096 15/30 Write block 17 len 20480 data: 10 9 9 9 8 16/30 Flush 17/30 Write block 118 len 4096 data: 10 18/30 Write block 357 len 24576 data: 12 12 12 12 12 9 19/30 Write block 433 len 28672 data: 8 9 9 9 12 12 8 20/30 Write block 255 len 36864 data: 4 5 8 8 8 7 8 6 7 21/30 Write block 122 len 32768 data: 7 7 8 6 7 8 7 6 22/30 Read block 20 len 40960 23/30 Read block 327 len 4096 24/30 Write block 352 len 24576 data: 8 7 8 12 13 13 25/30 Flush 26/30 Write block 267 len 8192 data: 5 7 27/30 Read block 181 len 16384 28/30 Write block 52 len 36864 data: 6 4 4 5 6 6 6 6 9 29/30 Write block 93 len 16384 data: 6 5 7 10 30/30 Read block 51 len 16384 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:32.778924537Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 99 100 172 169 172 F F F <--- 1 020-039 100 96 100 172 166 172 F F F <--- 2 040-059 98 98 98 168 168 168 T F T <--- 3 060-079 99 99 99 169 169 169 T F T <--- 4 080-099 97 97 97 167 167 167 T F T <--- 5 100-119 100 92 100 173 160 173 F F F <--- 6 120-139 100 99 100 173 169 173 F F F <--- 7 140-159 99 99 99 169 169 169 F F F 8 160-179 100 97 100 171 167 171 F F F <--- 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 100 93 100 173 162 173 F F F <--- 13 260-279 100 94 100 173 164 173 T F T <--- 14 280-299 97 97 97 167 167 167 F F F 15 300-319 98 98 98 168 168 168 F F F 16 320-339 100 98 100 171 168 171 F F F <--- 17 340-359 100 94 100 173 164 173 F F F <--- 18 360-379 100 93 100 173 162 173 F F F <--- 19 380-399 100 99 100 171 169 171 F F F <--- 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 94 100 173 164 173 F F F <--- 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 100 94 100 171 164 171 F F F <--- 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 94 100 172 164 172 F F F <--- 29 580-599 100 96 100 171 166 171 F F F <--- Max gen: 100, Max flush: 173 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 172 172 172 F F F 1 020-039 100 100 100 172 172 172 F F F 2 040-059 101 101 101 174 174 174 F F F 3 060-079 101 101 101 174 174 174 F F F 4 080-099 101 101 101 174 174 174 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 100 100 100 173 173 173 F F F 7 140-159 99 99 99 169 169 169 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 90 90 90 154 154 154 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 100 100 100 173 173 173 F F F 13 260-279 101 101 101 174 174 174 F F F 14 280-299 97 97 97 167 167 167 F F F 15 300-319 98 98 98 168 168 168 F F F 16 320-339 100 100 100 171 171 171 F F F 17 340-359 100 100 100 173 173 173 F F F 18 360-379 100 100 100 173 173 173 F F F 19 380-399 100 100 100 171 171 171 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 100 100 100 171 171 171 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 100 100 100 171 171 171 F F F Max gen: 101, Max flush: 174 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.113796364Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.125533712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.125892028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.125929965Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.125942224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.125960393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"Crucible 346d2eb9-91d7-47a2-b3d6-31b534f21259 has session id: 380ffb13-cf58-4fda-876a-9059ac2ac536","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126281461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Upstairs opts: Upstairs UUID: 346d2eb9-91d7-47a2-b3d6-31b534f21259, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126321019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Crucible stats registered with UUID: 346d2eb9-91d7-47a2-b3d6-31b534f21259","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126336588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:37.126352586Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126539004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126564362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126579171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126693293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126712452Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"346d2eb9-91d7-47a2-b3d6-31b534f21259 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126728661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.12674146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126755539Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126769408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126783437Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126797696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126811695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"ds_connection connected from Ok(127.0.0.1:53451)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126850562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"ds_connection connected from Ok(127.0.0.1:51667)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.126866981Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"ds_connection connected from Ok(127.0.0.1:58718)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.12688202Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.127599291Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.127628009Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.127645168Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.127975636Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128023502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[0]R flush_numbers[0..12]: [172, 172, 174, 174, 174, 173, 173, 169, 171, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128081898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[0]R generation[0..12]: [100, 100, 101, 101, 101, 100, 100, 99, 100, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128099047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128115716Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[1]R flush_numbers[0..12]: [172, 172, 174, 174, 174, 173, 173, 169, 171, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128131655Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[1]R generation[0..12]: [100, 100, 101, 101, 101, 100, 100, 99, 100, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128147184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128163273Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[2]R flush_numbers[0..12]: [172, 172, 174, 174, 174, 173, 173, 169, 171, 162, 154, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128178772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[2]R generation[0..12]: [100, 100, 101, 101, 101, 100, 100, 99, 100, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128194191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.12821048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128227368Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128247697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128261256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128274755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"":"downstairs","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128288674Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128300923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"0","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128314902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"1","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128328591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"client":"2","":"downstairs","session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.12834266Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128358629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"346d2eb9-91d7-47a2-b3d6-31b534f21259 is now active with session: 380ffb13-cf58-4fda-876a-9059ac2ac536","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128373988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128386777Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608,"session_id":"380ffb13-cf58-4fda-876a-9059ac2ac536"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128399937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} {"msg":"Activated sub_volume 346d2eb9-91d7-47a2-b3d6-31b534f21259","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:37.128411416Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 255 len 4096 data: 5 02/30 Read block 209 len 36864 03/30 Write block 38 len 36864 data: 13 14 12 9 9 8 7 6 7 04/30 Write block 13 len 4096 data: 9 05/30 Write block 334 len 8192 data: 5 7 06/30 Read block 504 len 32768 07/30 Read block 586 len 12288 08/30 Read block 417 len 24576 09/30 Read block 449 len 8192 10/30 Flush 11/30 Write block 532 len 28672 data: 8 9 9 6 6 5 5 12/30 Write block 140 len 4096 data: 9 13/30 Flush 14/30 Write block 96 len 16384 data: 11 10 9 9 15/30 Flush 16/30 Write block 260 len 20480 data: 8 9 7 8 8 17/30 Read block 386 len 12288 18/30 Write block 345 len 4096 data: 6 19/30 Write block 351 len 36864 data: 9 9 8 9 13 14 14 13 13 20/30 Read block 88 len 36864 21/30 Write block 537 len 4096 data: 6 22/30 Write block 201 len 40960 data: 11 11 11 12 11 12 11 12 13 12 23/30 Write block 15 len 28672 data: 9 8 11 10 10 10 9 24/30 Read block 289 len 8192 25/30 Read block 197 len 16384 26/30 Flush 27/30 Write block 580 len 16384 data: 5 7 7 8 28/30 Write block 303 len 4096 data: 8 29/30 Read block 350 len 12288 30/30 Write block 270 len 24576 data: 8 9 9 8 8 8 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 9 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:37.139953997Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 100 178 178 172 F F F <--- 1 020-039 102 102 100 178 178 172 F F F <--- 2 040-059 102 102 101 175 175 174 F F F <--- 3 060-079 101 101 101 174 174 174 F F F 4 080-099 102 102 101 177 177 174 F F F <--- 5 100-119 100 100 100 173 173 173 F F F 6 120-139 100 100 100 173 173 173 F F F 7 140-159 102 102 99 176 176 169 F F F <--- 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 90 178 178 154 F F F <--- 11 220-239 97 97 97 167 167 167 F F F 12 240-259 102 102 100 175 175 173 F F F <--- 13 260-279 102 102 101 178 178 174 T T F <--- 14 280-299 97 97 97 167 167 167 F F F 15 300-319 98 98 98 168 168 168 T T F <--- 16 320-339 102 102 100 175 175 171 F F F <--- 17 340-359 102 102 100 178 178 173 F F F <--- 18 360-379 100 100 100 173 173 173 F F F 19 380-399 100 100 100 171 171 171 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 102 102 100 178 178 171 F F F <--- 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 100 100 100 171 171 171 T T F <--- Max gen: 102, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 178 178 178 F F F 1 020-039 102 102 102 178 178 178 F F F 2 040-059 102 102 102 175 175 175 F F F 3 060-079 101 101 101 174 174 174 F F F 4 080-099 102 102 102 177 177 177 F F F 5 100-119 100 100 100 173 173 173 F F F 6 120-139 100 100 100 173 173 173 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 97 97 97 167 167 167 F F F 12 240-259 102 102 102 175 175 175 F F F 13 260-279 103 103 103 179 179 179 F F F 14 280-299 97 97 97 167 167 167 F F F 15 300-319 103 103 103 179 179 179 F F F 16 320-339 102 102 102 175 175 175 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 100 100 100 173 173 173 F F F 19 380-399 100 100 100 171 171 171 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 99 99 99 169 169 169 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 103 103 103 179 179 179 F F F Max gen: 103, Max flush: 179 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.463681047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:41.474396755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.474456651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.484938464Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485269832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485309419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485321518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485339307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"Crucible b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5 has session id: f253f71b-d7ea-459c-9088-6977fb70f3e6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485681994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Upstairs opts: Upstairs UUID: b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485716681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Crucible stats registered with UUID: b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.48573236Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:41.485746039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485930417Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485966684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.485982353Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486079786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486097455Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486121604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486134783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486148532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486162171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.48617572Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486190289Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486223507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37647)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486258324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35507)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486273923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58344)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.486289022Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487041271Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487067319Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487083908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487417215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487432764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[0]R flush_numbers[0..12]: [178, 178, 175, 174, 177, 173, 173, 176, 171, 162, 178, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487458992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[0]R generation[0..12]: [102, 102, 102, 101, 102, 100, 100, 102, 100, 93, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487475041Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.48749157Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[1]R flush_numbers[0..12]: [178, 178, 175, 174, 177, 173, 173, 176, 171, 162, 178, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487507449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[1]R generation[0..12]: [102, 102, 102, 101, 102, 100, 100, 102, 100, 93, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487523258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487539557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[2]R flush_numbers[0..12]: [178, 178, 175, 174, 177, 173, 173, 176, 171, 162, 178, 167]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487555186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[2]R generation[0..12]: [102, 102, 102, 101, 102, 100, 100, 102, 100, 93, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487570625Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487586694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487600243Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:35:41.487614682Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.48763321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487646529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"":"downstairs","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487660608Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487674718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"0","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487689117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"1","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487703286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"client":"2","":"downstairs","session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487717745Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487734443Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5 is now active with session: f253f71b-d7ea-459c-9088-6977fb70f3e6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487749292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487761822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616,"session_id":"f253f71b-d7ea-459c-9088-6977fb70f3e6"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.487775081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} {"msg":"Activated sub_volume b39316da-dfe2-4d0f-a9c7-20ce7f9e68a5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:41.48778686Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 16384 data: 10 10 10 10 02/30 Read block 442 len 24576 03/30 Write block 3 len 40960 data: 7 6 8 8 8 9 12 13 10 12 04/30 Flush 05/30 Write block 117 len 16384 data: 8 11 10 10 06/30 Read block 540 len 20480 07/30 Read block 482 len 40960 08/30 Flush 09/30 Write block 581 len 16384 data: 8 8 9 9 10/30 Read block 417 len 8192 11/30 Flush 12/30 Write block 24 len 20480 data: 11 8 6 8 8 13/30 Write block 220 len 20480 data: 10 9 11 12 13 14/30 Flush 15/30 Write block 50 len 20480 data: 8 7 7 5 5 16/30 Write block 315 len 20480 data: 9 8 9 11 9 17/30 Read block 432 len 20480 18/30 Flush 19/30 Write block 450 len 12288 data: 9 12 15 20/30 Read block 137 len 4096 21/30 Read block 480 len 40960 22/30 Write block 51 len 12288 data: 8 8 6 23/30 Flush 24/30 Write block 121 len 24576 data: 8 8 8 9 7 8 25/30 Write block 278 len 20480 data: 8 7 8 9 8 26/30 Read block 386 len 4096 27/30 Read block 141 len 40960 28/30 Write block 230 len 4096 data: 14 29/30 Read block 188 len 32768 30/30 Read block 191 len 28672 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 19 0 0 19 Done 0 19 19 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:41.499770731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 180 180 180 F F F 1 020-039 102 104 104 178 183 183 F F F <--- 2 040-059 102 104 104 175 185 185 F F F <--- 3 060-079 101 101 101 174 174 174 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 104 104 104 181 181 181 F T T <--- 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 97 104 104 167 183 183 F T T <--- 12 240-259 102 102 102 175 175 175 F F F 13 260-279 103 103 103 179 179 179 F T T <--- 14 280-299 97 97 97 167 167 167 F T T <--- 15 300-319 103 104 104 179 184 184 F F F <--- 16 320-339 102 102 102 175 175 175 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 100 100 100 173 173 173 F F F 19 380-399 100 100 100 171 171 171 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 99 104 104 169 185 185 F F F <--- 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 104 104 104 182 182 182 F F F Max gen: 104, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 180 180 180 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 185 185 185 F F F 3 060-079 101 101 101 174 174 174 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 105 105 105 186 186 186 F F F 12 240-259 102 102 102 175 175 175 F F F 13 260-279 105 105 105 186 186 186 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 104 104 104 184 184 184 F F F 16 320-339 102 102 102 175 175 175 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 100 100 100 173 173 173 F F F 19 380-399 100 100 100 171 171 171 F F F 20 400-419 98 98 98 168 168 168 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 104 104 104 185 185 185 F F F 23 460-479 99 99 99 169 169 169 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 97 97 167 167 167 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 99 99 99 169 169 169 F F F 28 560-579 100 100 100 172 172 172 F F F 29 580-599 104 104 104 182 182 182 F F F Max gen: 105, Max flush: 186 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.818330703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:45.82905807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.829116026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.839389135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.839684644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.839723502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.839735601Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.83975344Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"Crucible 00e42024-123b-40bf-bbd2-4e3323a7a53b has session id: 625ffdd0-d9d0-4736-b35d-bb9b670845af","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840096936Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Upstairs opts: Upstairs UUID: 00e42024-123b-40bf-bbd2-4e3323a7a53b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840134974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Crucible stats registered with UUID: 00e42024-123b-40bf-bbd2-4e3323a7a53b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840150872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:45.840164212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.84033255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840357458Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840372127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840512618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840537166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"00e42024-123b-40bf-bbd2-4e3323a7a53b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840549885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840562644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840577193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840591692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840605501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.8406199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840634169Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"ds_connection connected from Ok(127.0.0.1:62221)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840689196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"ds_connection connected from Ok(127.0.0.1:49239)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840707724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"ds_connection connected from Ok(127.0.0.1:56628)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.840722793Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841471362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.8415004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841517209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841856526Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841901713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[0]R flush_numbers[0..12]: [180, 183, 185, 174, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841952119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[0]R generation[0..12]: [104, 104, 104, 101, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841969508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.841998506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[1]R flush_numbers[0..12]: [180, 183, 185, 174, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842014665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[1]R generation[0..12]: [104, 104, 104, 101, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842030264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842046623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[2]R flush_numbers[0..12]: [180, 183, 185, 174, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842062192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[2]R generation[0..12]: [104, 104, 104, 101, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842080421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:35:45.84209685Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842115238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842129467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842142836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842156175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"":"downstairs","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842170254Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842182904Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"0","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842197083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"1","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842212652Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"client":"2","":"downstairs","session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842226991Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.84224035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"00e42024-123b-40bf-bbd2-4e3323a7a53b is now active with session: 625ffdd0-d9d0-4736-b35d-bb9b670845af","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842255239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842267848Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624,"session_id":"625ffdd0-d9d0-4736-b35d-bb9b670845af"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842280987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} {"msg":"Activated sub_volume 00e42024-123b-40bf-bbd2-4e3323a7a53b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:45.842294186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 36864 data: 9 9 8 11 10 11 12 11 11 02/30 Read block 505 len 28672 03/30 Read block 406 len 40960 04/30 Flush 05/30 Flush 06/30 Write block 26 len 40960 data: 7 9 9 11 12 11 11 9 9 9 07/30 Read block 99 len 4096 08/30 Read block 71 len 28672 09/30 Flush 10/30 Write block 60 len 28672 data: 10 10 11 13 15 14 14 11/30 Read block 278 len 40960 12/30 Read block 272 len 32768 13/30 Write block 468 len 12288 data: 8 8 10 14/30 Write block 11 len 16384 data: 11 13 10 9 15/30 Write block 517 len 24576 data: 11 12 10 10 8 7 16/30 Write block 249 len 12288 data: 9 8 7 17/30 Flush 18/30 Write block 268 len 20480 data: 8 8 9 10 10 19/30 Read block 97 len 40960 20/30 Read block 80 len 4096 21/30 Write block 266 len 40960 data: 7 6 9 9 10 11 11 9 9 9 22/30 Read block 247 len 16384 23/30 Write block 562 len 20480 data: 9 8 10 7 8 24/30 Read block 213 len 4096 25/30 Write block 558 len 4096 data: 11 26/30 Read block 57 len 8192 27/30 Write block 300 len 8192 data: 8 8 28/30 Read block 58 len 24576 29/30 Read block 194 len 16384 30/30 Write block 409 len 20480 data: 11 9 11 10 9 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:45.854607235Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 106 106 180 190 190 F F F <--- 1 020-039 104 106 106 183 189 189 F F F <--- 2 040-059 104 104 104 185 185 185 F F F 3 060-079 101 106 106 174 190 190 F F F <--- 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 105 105 105 186 186 186 F F F 12 240-259 102 106 106 175 190 190 F F F <--- 13 260-279 105 105 105 186 186 186 F T T <--- 14 280-299 105 105 105 186 186 186 F F F 15 300-319 104 104 104 184 184 184 F T T <--- 16 320-339 102 102 102 175 175 175 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 100 106 106 173 187 187 F F F <--- 19 380-399 100 106 106 171 187 187 F F F <--- 20 400-419 98 98 98 168 168 168 F T T <--- 21 420-439 100 100 100 173 173 173 F F F 22 440-459 104 104 104 185 185 185 F F F 23 460-479 99 106 106 169 190 190 F F F <--- 24 480-499 99 99 99 169 169 169 F F F 25 500-519 97 106 106 167 190 190 F F F <--- 26 520-539 102 106 106 178 190 190 F F F <--- 27 540-559 99 99 99 169 169 169 F T T <--- 28 560-579 100 100 100 172 172 172 F T T <--- 29 580-599 104 104 104 182 182 182 F F F Max gen: 106, Max flush: 190 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 106 106 106 189 189 189 F F F 2 040-059 104 104 104 185 185 185 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 105 105 105 186 186 186 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 107 107 107 191 191 191 F F F 16 320-339 102 102 102 175 175 175 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 106 106 106 187 187 187 F F F 20 400-419 107 107 107 191 191 191 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 104 104 104 185 185 185 F F F 23 460-479 106 106 106 190 190 190 F F F 24 480-499 99 99 99 169 169 169 F F F 25 500-519 106 106 106 190 190 190 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 107 107 107 191 191 191 F F F 28 560-579 107 107 107 191 191 191 F F F 29 580-599 104 104 104 182 182 182 F F F Max gen: 107, Max flush: 191 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.193072727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.204939957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205257435Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205295702Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205307661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.20532558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"Crucible 70181938-0d5d-44ad-ad59-46907c9b3a8c has session id: 97225e2c-67de-45fb-b92f-31caec296705","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205696765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Upstairs opts: Upstairs UUID: 70181938-0d5d-44ad-ad59-46907c9b3a8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205731573Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Crucible stats registered with UUID: 70181938-0d5d-44ad-ad59-46907c9b3a8c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205747181Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:50.205760461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205936379Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205961237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.205976276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206097348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206114476Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"70181938-0d5d-44ad-ad59-46907c9b3a8c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206131345Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206144174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206158013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206171832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"2","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.20620035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206248247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206273365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"ds_connection connected from Ok(127.0.0.1:58065)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206289804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"ds_connection connected from Ok(127.0.0.1:50557)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206304973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"ds_connection connected from Ok(127.0.0.1:60933)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.206319792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207010195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207036013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207052372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"2","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.20737725Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207392969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[0]R flush_numbers[0..12]: [190, 189, 185, 190, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207431246Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[0]R generation[0..12]: [106, 106, 104, 106, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207448195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207464424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[1]R flush_numbers[0..12]: [190, 189, 185, 190, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207479953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[1]R generation[0..12]: [106, 106, 104, 106, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207495572Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207511791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[2]R flush_numbers[0..12]: [190, 189, 185, 190, 180, 181, 186, 176, 171, 162, 178, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.20752742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[2]R generation[0..12]: [106, 106, 104, 106, 104, 104, 105, 102, 100, 93, 102, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207542959Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207558838Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-16T21:35:50.207571877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207600735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207614084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207627523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"":"downstairs","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207641402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207655661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"0","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.20766983Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"1","":"downstairs","session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207683369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207700008Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207713447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"70181938-0d5d-44ad-ad59-46907c9b3a8c is now active with session: 97225e2c-67de-45fb-b92f-31caec296705","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207728146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207740365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632,"session_id":"97225e2c-67de-45fb-b92f-31caec296705"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207753624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} {"msg":"Activated sub_volume 70181938-0d5d-44ad-ad59-46907c9b3a8c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:50.207764854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 4096 data: 8 02/30 Write block 542 len 20480 data: 9 9 8 7 7 03/30 Read block 78 len 28672 04/30 Write block 329 len 12288 data: 6 6 6 05/30 Write block 504 len 24576 data: 17 16 15 16 13 12 06/30 Read block 374 len 16384 07/30 Read block 489 len 12288 08/30 Read block 101 len 16384 09/30 Write block 386 len 36864 data: 13 13 12 14 13 12 12 14 12 10/30 Read block 54 len 28672 11/30 Read block 572 len 8192 12/30 Read block 368 len 24576 13/30 Read block 445 len 16384 14/30 Read block 49 len 24576 15/30 Write block 335 len 8192 data: 8 9 16/30 Read block 482 len 36864 17/30 Read block 221 len 16384 18/30 Read block 71 len 36864 19/30 Write block 222 len 32768 data: 12 13 14 12 12 14 13 12 20/30 Read block 98 len 20480 21/30 Read block 87 len 28672 22/30 Read block 401 len 16384 23/30 Flush 24/30 Flush 25/30 Write block 408 len 28672 data: 10 12 10 12 11 10 10 26/30 Read block 250 len 28672 27/30 Write block 568 len 16384 data: 5 3 6 7 28/30 Read block 338 len 24576 29/30 Write block 495 len 8192 data: 11 8 30/30 Read block 377 len 40960 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:50.21981801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 108 108 106 192 192 189 F F T <--- 2 040-059 104 104 104 185 185 185 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 108 108 105 192 192 186 F F F <--- 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 107 107 107 191 191 191 F F F 16 320-339 108 108 102 192 192 175 F F F <--- 17 340-359 102 102 102 178 178 178 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 108 108 106 192 192 187 F F F <--- 20 400-419 107 107 107 191 191 191 T T F <--- 21 420-439 100 100 100 173 173 173 F F F 22 440-459 104 104 104 185 185 185 F F F 23 460-479 106 106 106 190 190 190 F F F 24 480-499 99 99 99 169 169 169 T T F <--- 25 500-519 108 108 106 192 192 190 F F F <--- 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 107 192 192 191 F F F <--- 28 560-579 107 107 107 191 191 191 T T F <--- 29 580-599 104 104 104 182 182 182 F F F Max gen: 108, Max flush: 192 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 108 108 108 192 192 192 F F F 2 040-059 104 104 104 185 185 185 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 102 102 176 176 176 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 108 108 108 192 192 192 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 107 107 107 191 191 191 F F F 16 320-339 108 108 108 192 192 192 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 106 106 106 187 187 187 F F F 19 380-399 108 108 108 192 192 192 F F F 20 400-419 109 109 109 193 193 193 F F F 21 420-439 100 100 100 173 173 173 F F F 22 440-459 104 104 104 185 185 185 F F F 23 460-479 106 106 106 190 190 190 F F F 24 480-499 109 109 109 193 193 193 F F F 25 500-519 108 108 108 192 192 192 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 108 192 192 192 F F F 28 560-579 109 109 109 193 193 193 F F F 29 580-599 104 104 104 182 182 182 F F F Max gen: 109, Max flush: 193 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.53299938Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:54.543734057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.543783354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55407611Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554382329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554419957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554432406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554454124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"Crucible 989184e4-bc4b-424c-ae57-c027c4e15ab6 has session id: 5731a684-df11-4195-94dd-26a83b156061","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55481671Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Upstairs opts: Upstairs UUID: 989184e4-bc4b-424c-ae57-c027c4e15ab6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554850767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Crucible stats registered with UUID: 989184e4-bc4b-424c-ae57-c027c4e15ab6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.554868036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:54.554881505Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555042874Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555070752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555085421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555202033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555220262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"989184e4-bc4b-424c-ae57-c027c4e15ab6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555235631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55524837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555269169Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555283178Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555296957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555314146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555328575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"ds_connection connected from Ok(127.0.0.1:52308)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555353263Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"ds_connection connected from Ok(127.0.0.1:45287)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555368112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"ds_connection connected from Ok(127.0.0.1:43778)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.555453536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55612591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556155038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556178497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556515454Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556529593Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[0]R flush_numbers[0..12]: [190, 192, 185, 190, 180, 181, 186, 176, 171, 162, 178, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55656874Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[0]R generation[0..12]: [106, 108, 104, 106, 104, 104, 105, 102, 100, 93, 102, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556585989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556602188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[1]R flush_numbers[0..12]: [190, 192, 185, 190, 180, 181, 186, 176, 171, 162, 178, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556617807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[1]R generation[0..12]: [106, 108, 104, 106, 104, 104, 105, 102, 100, 93, 102, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556633116Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556652114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[2]R flush_numbers[0..12]: [190, 192, 185, 190, 180, 181, 186, 176, 171, 162, 178, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556667863Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[2]R generation[0..12]: [106, 108, 104, 106, 104, 104, 105, 102, 100, 93, 102, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556683402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556699571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.5567133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556727579Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556746908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556760367Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"":"downstairs","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556774696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556787455Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"0","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556801794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"1","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556815923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"client":"2","":"downstairs","session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556830172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556843401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"989184e4-bc4b-424c-ae57-c027c4e15ab6 is now active with session: 5731a684-df11-4195-94dd-26a83b156061","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.55686348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556876729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640,"session_id":"5731a684-df11-4195-94dd-26a83b156061"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556890198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} {"msg":"Activated sub_volume 989184e4-bc4b-424c-ae57-c027c4e15ab6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:54.556901797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 4096 data: 10 02/30 Write block 140 len 32768 data: 10 9 8 6 8 7 5 5 03/30 Read block 480 len 28672 04/30 Read block 149 len 20480 05/30 Read block 537 len 20480 06/30 Write block 440 len 32768 data: 7 8 7 8 6 8 10 8 07/30 Flush 08/30 Flush 09/30 Write block 563 len 8192 data: 9 11 10/30 Write block 307 len 32768 data: 7 7 7 7 7 6 8 9 11/30 Write block 441 len 12288 data: 9 8 9 12/30 Write block 32 len 4096 data: 12 13/30 Read block 487 len 12288 14/30 Write block 374 len 24576 data: 9 12 11 12 13 12 15/30 Write block 314 len 40960 data: 10 10 9 10 12 10 9 9 9 7 16/30 Read block 489 len 4096 17/30 Write block 48 len 8192 data: 8 8 18/30 Read block 96 len 20480 19/30 Read block 254 len 24576 20/30 Write block 223 len 20480 data: 14 15 13 13 15 21/30 Read block 141 len 8192 22/30 Read block 530 len 36864 23/30 Read block 354 len 28672 24/30 Write block 467 len 16384 data: 9 9 9 11 25/30 Read block 4 len 16384 26/30 Read block 204 len 12288 27/30 Write block 597 len 12288 data: 4 3 2 28/30 Write block 431 len 20480 data: 13 11 9 10 10 29/30 Write block 32 len 28672 data: 13 10 10 10 9 9 14 30/30 Write block 448 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 5 Sent Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 2 35 Done 0 27 28 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:54.567867698Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 108 110 110 192 194 194 F T T <--- 2 040-059 104 104 104 185 185 185 F T T <--- 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 102 110 110 176 194 194 F F F <--- 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 108 108 108 192 192 192 F T T <--- 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 107 107 107 191 191 191 F T T <--- 16 320-339 108 108 108 192 192 192 F T T <--- 17 340-359 102 102 102 178 178 178 F F F 18 360-379 106 106 106 187 187 187 F T T <--- 19 380-399 108 108 108 192 192 192 F F F 20 400-419 109 109 109 193 193 193 F F F 21 420-439 100 100 100 173 173 173 F T T <--- 22 440-459 104 110 110 185 194 194 F T T <--- 23 460-479 106 106 106 190 190 190 F T T <--- 24 480-499 109 109 109 193 193 193 F F F 25 500-519 108 108 108 192 192 192 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 108 192 192 192 F F F 28 560-579 109 109 109 193 193 193 F T T <--- 29 580-599 104 104 104 182 182 182 F T T <--- Max gen: 110, Max flush: 194 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 111 111 111 195 195 195 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 F F F 5 100-119 104 104 104 181 181 181 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 100 100 100 171 171 171 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 111 111 111 195 195 195 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 111 111 111 195 195 195 F F F 16 320-339 111 111 111 195 195 195 F F F 17 340-359 102 102 102 178 178 178 F F F 18 360-379 111 111 111 195 195 195 F F F 19 380-399 108 108 108 192 192 192 F F F 20 400-419 109 109 109 193 193 193 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 111 111 111 195 195 195 F F F 24 480-499 109 109 109 193 193 193 F F F 25 500-519 108 108 108 192 192 192 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 108 192 192 192 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 111, Max flush: 195 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.90179519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.913721465Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91409034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914129248Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914141677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914159325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"Crucible a8942cfc-b247-4029-947b-6fa2b4b7d0d6 has session id: 4bdf1138-d427-4b02-b532-30f13bc3cf44","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914488503Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Upstairs opts: Upstairs UUID: a8942cfc-b247-4029-947b-6fa2b4b7d0d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914522801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Crucible stats registered with UUID: a8942cfc-b247-4029-947b-6fa2b4b7d0d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91453801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:58.914551809Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914731236Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914760604Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914775853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914879406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914897765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"a8942cfc-b247-4029-947b-6fa2b4b7d0d6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914928003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914941212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914955511Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91496955Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914983609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.914997808Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915044325Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:38608)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915071853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:48606)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915088032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:54506)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915231012Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915762106Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.915792944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91585503Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916164318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916179587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[0]R flush_numbers[0..12]: [190, 195, 195, 190, 180, 181, 186, 194, 171, 162, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916270961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[0]R generation[0..12]: [106, 111, 111, 106, 104, 104, 105, 110, 100, 93, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91628781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916304529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[1]R flush_numbers[0..12]: [190, 195, 195, 190, 180, 181, 186, 194, 171, 162, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916319878Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[1]R generation[0..12]: [106, 111, 111, 106, 104, 104, 105, 110, 100, 93, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916335327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916351586Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[2]R flush_numbers[0..12]: [190, 195, 195, 190, 180, 181, 186, 194, 171, 162, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916367455Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[2]R generation[0..12]: [106, 111, 111, 106, 104, 104, 105, 110, 100, 93, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916383144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916399512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916412911Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91643555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916449029Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916462508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"":"downstairs","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916476847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916489536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"0","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916504065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"1","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2026-03-16T21:35:58.916520714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"client":"2","":"downstairs","session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916535333Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916548782Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"a8942cfc-b247-4029-947b-6fa2b4b7d0d6 is now active with session: 4bdf1138-d427-4b02-b532-30f13bc3cf44","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916563691Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.91657656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648,"session_id":"4bdf1138-d427-4b02-b532-30f13bc3cf44"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916589829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} {"msg":"Activated sub_volume a8942cfc-b247-4029-947b-6fa2b4b7d0d6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:35:58.916601369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 322 len 12288 data: 10 8 8 02/30 Write block 381 len 40960 data: 8 9 11 12 12 14 14 13 15 14 03/30 Read block 234 len 12288 04/30 Read block 84 len 40960 05/30 Read block 359 len 16384 06/30 Write block 353 len 28672 data: 9 10 14 15 15 14 14 07/30 Write block 114 len 12288 data: 8 7 8 08/30 Read block 585 len 12288 09/30 Read block 276 len 4096 10/30 Read block 464 len 12288 11/30 Write block 394 len 36864 data: 13 14 14 13 14 12 11 8 9 12/30 Read block 63 len 8192 13/30 Read block 522 len 20480 14/30 Read block 446 len 12288 15/30 Read block 213 len 36864 16/30 Read block 195 len 16384 17/30 Read block 448 len 12288 18/30 Read block 586 len 28672 19/30 Write block 476 len 16384 data: 5 6 10 11 20/30 Read block 232 len 4096 21/30 Read block 543 len 12288 22/30 Write block 81 len 28672 data: 7 6 5 6 5 4 4 23/30 Read block 13 len 8192 24/30 Read block 528 len 12288 25/30 Read block 280 len 12288 26/30 Read block 344 len 28672 27/30 Read block 239 len 32768 28/30 Write block 169 len 16384 data: 10 11 8 10 29/30 Read block 198 len 36864 30/30 Write block 179 len 32768 data: 7 9 10 11 9 8 7 6 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:35:58.928539743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 111 111 111 195 195 195 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 104 104 104 180 180 180 T F T <--- 5 100-119 104 104 104 181 181 181 T F T <--- 6 120-139 105 105 105 186 186 186 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 100 100 100 171 171 171 T F T <--- 9 180-199 93 93 93 162 162 162 T F T <--- 10 200-219 102 102 102 178 178 178 F F F 11 220-239 111 111 111 195 195 195 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 111 111 111 195 195 195 F F F 16 320-339 111 111 111 195 195 195 T T T 17 340-359 102 102 102 178 178 178 T F T <--- 18 360-379 111 111 111 195 195 195 F F F 19 380-399 108 108 108 192 192 192 T F T <--- 20 400-419 109 109 109 193 193 193 T F T <--- 21 420-439 111 111 111 195 195 195 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 111 111 111 195 195 195 T F T <--- 24 480-499 109 109 109 193 193 193 F F F 25 500-519 108 108 108 192 192 192 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 108 192 192 192 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 111, Max flush: 195 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 111 111 111 195 195 195 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 106 106 106 190 190 190 F F F 4 080-099 112 112 112 196 196 196 F F F 5 100-119 112 112 112 196 196 196 F F F 6 120-139 105 105 105 186 186 186 F F F 7 140-159 110 110 110 194 194 194 F F F 8 160-179 112 112 112 196 196 196 F F F 9 180-199 112 112 112 196 196 196 F F F 10 200-219 102 102 102 178 178 178 F F F 11 220-239 111 111 111 195 195 195 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 105 105 105 186 186 186 F F F 15 300-319 111 111 111 195 195 195 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 112 112 112 196 196 196 F F F 18 360-379 111 111 111 195 195 195 F F F 19 380-399 112 112 112 196 196 196 F F F 20 400-419 112 112 112 196 196 196 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 111 111 111 195 195 195 F F F 23 460-479 112 112 112 196 196 196 F F F 24 480-499 109 109 109 193 193 193 F F F 25 500-519 108 108 108 192 192 192 F F F 26 520-539 106 106 106 190 190 190 F F F 27 540-559 108 108 108 192 192 192 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 112, Max flush: 196 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.245830583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.257584189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.257945005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.257984172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.257999081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.25801727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"Crucible e0fa118b-5cea-417d-842b-50acf102af7d has session id: d0349e09-60ee-4116-8fb2-719d19f50695","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258357497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Upstairs opts: Upstairs UUID: e0fa118b-5cea-417d-842b-50acf102af7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258394054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Crucible stats registered with UUID: e0fa118b-5cea-417d-842b-50acf102af7d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258409853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:03.258423182Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258612659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258648217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258664076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258775068Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258793337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"e0fa118b-5cea-417d-842b-50acf102af7d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258810686Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258823535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258837614Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258851643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258867772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258882411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.25889665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"ds_connection connected from Ok(127.0.0.1:33627)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258937047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"ds_connection connected from Ok(127.0.0.1:54795)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.258953396Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"ds_connection connected from Ok(127.0.0.1:62715)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.25904453Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.259732993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.259761051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.259783769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260129056Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260179382Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[0]R flush_numbers[0..12]: [190, 195, 195, 190, 196, 196, 186, 194, 196, 196, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260239108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[0]R generation[0..12]: [106, 111, 111, 106, 112, 112, 105, 110, 112, 112, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260256247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260272546Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[1]R flush_numbers[0..12]: [190, 195, 195, 190, 196, 196, 186, 194, 196, 196, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260291135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[1]R generation[0..12]: [106, 111, 111, 106, 112, 112, 105, 110, 112, 112, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260307123Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260323222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[2]R flush_numbers[0..12]: [190, 195, 195, 190, 196, 196, 186, 194, 196, 196, 178, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260338861Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[2]R generation[0..12]: [106, 111, 111, 106, 112, 112, 105, 110, 112, 112, 102, 111]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.26035424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260370309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:36:03.260383868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260403117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260416446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260429525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"":"downstairs","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260443734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260456003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"0","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260469932Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"1","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260483731Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"client":"2","":"downstairs","session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.26050087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260516019Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"e0fa118b-5cea-417d-842b-50acf102af7d is now active with session: d0349e09-60ee-4116-8fb2-719d19f50695","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260531188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260543987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656,"session_id":"d0349e09-60ee-4116-8fb2-719d19f50695"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260557136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} {"msg":"Activated sub_volume e0fa118b-5cea-417d-842b-50acf102af7d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:03.260568676Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 515 len 40960 data: 9 10 12 13 11 11 9 8 4 4 02/30 Read block 543 len 28672 03/30 Read block 205 len 4096 04/30 Write block 456 len 28672 data: 14 14 12 13 12 8 7 05/30 Write block 400 len 40960 data: 12 9 10 8 9 9 11 12 11 13 06/30 Write block 68 len 28672 data: 14 12 9 9 8 10 7 07/30 Read block 468 len 20480 08/30 Write block 351 len 28672 data: 10 10 10 11 15 16 16 09/30 Read block 486 len 4096 10/30 Read block 134 len 36864 11/30 Write block 452 len 36864 data: 16 13 13 11 15 15 13 14 13 12/30 Write block 151 len 16384 data: 8 8 7 5 13/30 Write block 312 len 24576 data: 7 9 11 11 10 11 14/30 Flush 15/30 Write block 36 len 8192 data: 10 10 16/30 Write block 480 len 40960 data: 12 12 12 11 8 9 12 15 16 13 17/30 Write block 125 len 32768 data: 8 9 9 8 7 7 6 7 18/30 Read block 400 len 4096 19/30 Write block 379 len 12288 data: 13 12 9 20/30 Read block 380 len 32768 21/30 Read block 282 len 36864 22/30 Write block 147 len 12288 data: 6 6 8 23/30 Write block 177 len 40960 data: 8 7 8 10 11 12 10 9 8 7 24/30 Read block 153 len 28672 25/30 Write block 295 len 28672 data: 6 7 9 9 7 9 9 26/30 Flush 27/30 Write block 215 len 40960 data: 6 7 8 7 8 11 10 13 15 16 28/30 Write block 557 len 12288 data: 10 12 10 29/30 Write block 537 len 12288 data: 7 6 6 30/30 Read block 413 len 24576 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:03.274011357Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 114 114 111 198 198 195 F F F <--- 2 040-059 111 111 111 195 195 195 F F F 3 060-079 114 114 106 197 197 190 F F F <--- 4 080-099 112 112 112 196 196 196 F F F 5 100-119 112 112 112 196 196 196 F F F 6 120-139 114 114 105 198 198 186 F F F <--- 7 140-159 114 114 110 198 198 194 F F F <--- 8 160-179 114 114 112 198 198 196 F F F <--- 9 180-199 114 114 112 198 198 196 F F F <--- 10 200-219 102 102 102 178 178 178 T T F <--- 11 220-239 111 111 111 195 195 195 T T F <--- 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 114 114 105 198 198 186 F F F <--- 15 300-319 114 114 111 198 198 195 F F F <--- 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 112 197 197 196 F F F <--- 18 360-379 114 114 111 198 198 195 F F F <--- 19 380-399 114 114 112 198 198 196 F F F <--- 20 400-419 114 114 112 197 197 196 F F F <--- 21 420-439 111 111 111 195 195 195 F F F 22 440-459 114 114 111 197 197 195 F F F <--- 23 460-479 114 114 112 197 197 196 F F F <--- 24 480-499 114 114 109 198 198 193 F F F <--- 25 500-519 114 114 108 197 197 192 F F F <--- 26 520-539 114 114 106 197 197 190 T T F <--- 27 540-559 108 108 108 192 192 192 T T F <--- 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 114, Max flush: 198 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 112 112 112 196 196 196 F F F 5 100-119 112 112 112 196 196 196 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 114 114 114 198 198 198 F F F 8 160-179 114 114 114 198 198 198 F F F 9 180-199 114 114 114 198 198 198 F F F 10 200-219 115 115 115 199 199 199 F F F 11 220-239 115 115 115 199 199 199 F F F 12 240-259 106 106 106 190 190 190 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 114 114 114 198 198 198 F F F 15 300-319 114 114 114 198 198 198 F F F 16 320-339 112 112 112 196 196 196 F F F 17 340-359 114 114 114 197 197 197 F F F 18 360-379 114 114 114 198 198 198 F F F 19 380-399 114 114 114 198 198 198 F F F 20 400-419 114 114 114 197 197 197 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 114 114 114 197 197 197 F F F 23 460-479 114 114 114 197 197 197 F F F 24 480-499 114 114 114 198 198 198 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 115, Max flush: 199 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.616220504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:07.62696291Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.627024186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.637467742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.637766342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.637805839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.637818028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.637835607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"Crucible 86d4e6c5-ed11-4825-a2e8-ed99763dd31a has session id: d873644c-a4c3-4947-9f9a-2f54e62558fd","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638178224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Upstairs opts: Upstairs UUID: 86d4e6c5-ed11-4825-a2e8-ed99763dd31a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638241189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Crucible stats registered with UUID: 86d4e6c5-ed11-4825-a2e8-ed99763dd31a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638260728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:07.638274127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638459155Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638496232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638512021Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638609074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638628373Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"86d4e6c5-ed11-4825-a2e8-ed99763dd31a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638645352Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638658051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.63867245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638686629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638700418Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638714817Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638729026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61443)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638771153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38779)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638796611Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62681)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.638882706Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.639567509Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.639609126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.639626255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.63999426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640033337Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[0]R flush_numbers[0..12]: [190, 198, 195, 197, 196, 196, 198, 198, 198, 198, 199, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640079394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[0]R generation[0..12]: [106, 114, 111, 114, 112, 112, 114, 114, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640096563Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640113551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[1]R flush_numbers[0..12]: [190, 198, 195, 197, 196, 196, 198, 198, 198, 198, 199, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.64012924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[1]R generation[0..12]: [106, 114, 111, 114, 112, 112, 114, 114, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640144869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640161498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[2]R flush_numbers[0..12]: [190, 198, 195, 197, 196, 196, 198, 198, 198, 198, 199, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640177367Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[2]R generation[0..12]: [106, 114, 111, 114, 112, 112, 114, 114, 114, 114, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640196066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640214035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msgWait for a query_work_queue command to finish before sending IO ":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640233833Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640248172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640261051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.64027429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"":"downstairs","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640289189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640302009Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"0","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640317418Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"1","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2026-03-16T21:36:07.640331607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"client":"2","":"downstairs","session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640349275Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640363034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"86d4e6c5-ed11-4825-a2e8-ed99763dd31a is now active with session: d873644c-a4c3-4947-9f9a-2f54e62558fd","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640378773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640391632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664,"session_id":"d873644c-a4c3-4947-9f9a-2f54e62558fd"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640405132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} {"msg":"Activated sub_volume 86d4e6c5-ed11-4825-a2e8-ed99763dd31a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:07.640416841Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 465 len 40960 data: 8 9 10 10 10 12 11 10 8 7 02/30 Write block 335 len 24576 data: 9 10 9 7 7 6 03/30 Read block 422 len 4096 04/30 Read block 359 len 4096 05/30 Write block 465 len 8192 data: 9 10 06/30 Read block 448 len 28672 07/30 Write block 407 len 24576 data: 13 12 14 11 13 12 08/30 Read block 318 len 36864 09/30 Read block 276 len 8192 10/30 Read block 289 len 4096 11/30 Read block 308 len 28672 12/30 Read block 114 len 16384 13/30 Write block 69 len 12288 data: 13 10 10 14/30 Read block 323 len 4096 15/30 Write block 318 len 40960 data: 13 11 10 10 11 9 9 7 7 8 16/30 Write block 234 len 40960 data: 12 13 13 10 8 7 6 7 11 10 17/30 Write block 297 len 4096 data: 10 18/30 Write block 478 len 12288 data: 11 12 13 19/30 Read block 253 len 24576 20/30 Read block 219 len 4096 21/30 Read block 536 len 12288 22/30 Read block 227 len 4096 23/30 Write block 69 len 36864 data: 14 11 11 9 11 8 7 7 7 24/30 Read block 134 len 4096 25/30 Read block 189 len 12288 26/30 Write block 316 len 8192 data: 11 12 27/30 Read block 302 len 12288 28/30 Write block 443 len 12288 data: 10 7 9 29/30 Write block 157 len 24576 data: 7 6 4 7 10 10 30/30 Read block 311 len 24576 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:07.651638004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 114 114 114 197 197 197 F T T <--- 4 080-099 112 112 112 196 196 196 F F F 5 100-119 112 112 112 196 196 196 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 114 114 114 198 198 198 F T T <--- 8 160-179 114 114 114 198 198 198 F T T <--- 9 180-199 114 114 114 198 198 198 F F F 10 200-219 115 115 115 199 199 199 F F F 11 220-239 115 115 115 199 199 199 F T T <--- 12 240-259 106 106 106 190 190 190 F T T <--- 13 260-279 107 107 107 191 191 191 F F F 14 280-299 114 114 114 198 198 198 F T T <--- 15 300-319 114 114 114 198 198 198 F T T <--- 16 320-339 112 112 112 196 196 196 F T T <--- 17 340-359 114 114 114 197 197 197 F T T <--- 18 360-379 114 114 114 198 198 198 F F F 19 380-399 114 114 114 198 198 198 F F F 20 400-419 114 114 114 197 197 197 F T T <--- 21 420-439 111 111 111 195 195 195 F F F 22 440-459 114 114 114 197 197 197 F T T <--- 23 460-479 114 114 114 197 197 197 T T T 24 480-499 114 114 114 198 198 198 F T T <--- 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 115, Max flush: 199 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 468d467 < 8, 472a472 > 10, 481d480 < 10, 482a482 > 12, diff found after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 190 190 190 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 111 111 111 195 195 195 F F F 3 060-079 116 116 116 200 200 200 F F F 4 080-099 112 112 112 196 196 196 F F F 5 100-119 112 112 112 196 196 196 F F F 6 120-139 114 114 114 198 198 198 F F F 7 140-159 116 116 116 200 200 200 F F F 8 160-179 116 116 116 200 200 200 F F F 9 180-199 114 114 114 198 198 198 F F F 10 200-219 115 115 115 199 199 199 F F F 11 220-239 116 116 116 200 200 200 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 107 107 107 191 191 191 F F F 14 280-299 116 116 116 200 200 200 F F F 15 300-319 116 116 116 200 200 200 F F F 16 320-339 116 116 116 200 200 200 F F F 17 340-359 116 116 116 200 200 200 F F F 18 360-379 114 114 114 198 198 198 F F F 19 380-399 114 114 114 198 198 198 F F F 20 400-419 116 116 116 200 200 200 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 116 116 116 200 200 200 F F F 23 460-479 116 116 116 200 200 200 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 F F F 28 560-579 111 111 111 195 195 195 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 116, Max flush: 200 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:11.99971234Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.011529312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.01186086Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.011903197Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.011916266Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.011934335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"Crucible 3e36d2a1-d465-4f2c-a2c9-257bb3777d22 has session id: 2975f337-4dd2-430d-8022-83f28a655dc3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012244533Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Upstairs opts: Upstairs UUID: 3e36d2a1-d465-4f2c-a2c9-257bb3777d22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012276301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Crucible stats registered with UUID: 3e36d2a1-d465-4f2c-a2c9-257bb3777d22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.01229117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:12.012304709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012507945Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012544363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012559752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012674094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012692953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"3e36d2a1-d465-4f2c-a2c9-257bb3777d22 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012709282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012722311Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.01273641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012750549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012764298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012778657Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012792976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:59330)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012813744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39079)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012828523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39706)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.012971424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.013555894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.013584042Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.013603541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.013939518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.013978765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[0]R flush_numbers[0..12]: [190, 198, 195, 200, 196, 196, 198, 200, 200, 198, 199, 200]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014059049Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[0]R generation[0..12]: [106, 114, 111, 116, 112, 112, 114, 116, 116, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014077548Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014094997Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[1]R flush_numbers[0..12]: [190, 198, 195, 200, 196, 196, 198, 200, 200, 198, 199, 200]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014111046Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[1]R generation[0..12]: [106, 114, 111, 116, 112, 112, 114, 116, 116, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014127495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014143894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[2]R flush_numbers[0..12]: [190, 198, 195, 200, 196, 196, 198, 200, 200, 198, 199, 200]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014159423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[2]R generation[0..12]: [106, 114, 111, 116, 112, 112, 114, 116, 116, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014175151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.01419163Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 117","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014211199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Generation requested: 118 >= found:117","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014225468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014238367Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014251806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"":"downstairs","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014268005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014281024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"0","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014295593Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"1","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-16T21:36:12.014309582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"client":"2","":"downstairs","session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014327131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.01434066Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"3e36d2a1-d465-4f2c-a2c9-257bb3777d22 is now active with session: 2975f337-4dd2-430d-8022-83f28a655dc3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014355699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014368438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675,"session_id":"2975f337-4dd2-430d-8022-83f28a655dc3"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014381767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} {"msg":"Activated sub_volume 3e36d2a1-d465-4f2c-a2c9-257bb3777d22","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:12.014393227Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 6 len 40960 data: 9 9 10 13 14 12 14 11 10 10 02/30 Read block 519 len 32768 03/30 Write block 281 len 16384 data: 10 9 7 9 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 560 len 36864 data: 10 11 10 10 12 8 9 6 6 08/30 Write block 567 len 36864 data: 7 7 4 7 8 7 5 8 7 09/30 Write block 396 len 12288 data: 15 14 15 10/30 Write block 105 len 8192 data: 7 7 11/30 Write block 402 len 16384 data: 11 9 10 10 12/30 Read block 299 len 28672 13/30 Write block 119 len 32768 data: 11 11 9 9 9 10 9 10 14/30 Flush 15/30 Write block 56 len 40960 data: 7 7 7 7 11 11 12 14 16 15 16/30 Write block 346 len 20480 data: 9 9 8 9 9 17/30 Read block 593 len 24576 18/30 Write block 215 len 24576 data: 7 8 9 8 9 12 19/30 Read block 226 len 24576 20/30 Read block 575 len 16384 21/30 Flush 22/30 Write block 281 len 36864 data: 11 10 8 10 9 8 8 9 7 23/30 Read block 139 len 28672 24/30 Flush 25/30 Flush 26/30 Write block 134 len 40960 data: 9 9 13 10 12 9 11 10 9 7 27/30 Read block 429 len 36864 28/30 Write block 332 len 40960 data: 7 6 6 10 11 10 8 8 7 8 29/30 Write block 272 len 8192 data: 12 10 30/30 Read block 97 len 8192 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Flush 0 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 26 28 Done 26 24 0 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:12.027837558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1675} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 201 201 201 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 118 118 111 205 205 195 F F F <--- 3 060-079 118 118 116 205 205 200 F F F <--- 4 080-099 112 112 112 196 196 196 F F F 5 100-119 118 118 112 204 204 196 F F F <--- 6 120-139 118 118 114 204 204 198 T T F <--- 7 140-159 116 116 116 200 200 200 T T F <--- 8 160-179 116 116 116 200 200 200 F F F 9 180-199 114 114 114 198 198 198 F F F 10 200-219 118 118 115 205 205 199 F F F <--- 11 220-239 118 118 116 205 205 200 F F F <--- 12 240-259 116 116 116 200 200 200 F F F 13 260-279 107 107 107 191 191 191 T T F <--- 14 280-299 118 118 118 206 206 201 F F F <--- 15 300-319 116 116 116 200 200 200 F F F 16 320-339 116 116 116 200 200 200 T T F <--- 17 340-359 118 118 116 205 205 200 T T F <--- 18 360-379 114 114 114 198 198 198 F F F 19 380-399 118 118 114 204 204 198 F F F <--- 20 400-419 118 118 116 204 204 200 F F F <--- 21 420-439 111 111 111 195 195 195 F F F 22 440-459 116 116 116 200 200 200 F F F 23 460-479 116 116 116 200 200 200 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 F F F 28 560-579 118 118 111 204 204 195 F F F <--- 29 580-599 111 111 111 195 195 195 F F F Max gen: 118, Max flush: 206 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 201 201 201 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 118 118 118 205 205 205 F F F 3 060-079 118 118 118 205 205 205 F F F 4 080-099 112 112 112 196 196 196 F F F 5 100-119 118 118 118 204 204 204 F F F 6 120-139 119 119 119 207 207 207 F F F 7 140-159 119 119 119 207 207 207 F F F 8 160-179 116 116 116 200 200 200 F F F 9 180-199 114 114 114 198 198 198 F F F 10 200-219 118 118 118 205 205 205 F F F 11 220-239 118 118 118 205 205 205 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 119 119 119 207 207 207 F F F 14 280-299 118 118 118 206 206 206 F F F 15 300-319 116 116 116 200 200 200 F F F 16 320-339 119 119 119 207 207 207 F F F 17 340-359 119 119 119 207 207 207 F F F 18 360-379 114 114 114 198 198 198 F F F 19 380-399 118 118 118 204 204 204 F F F 20 400-419 118 118 118 204 204 204 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 116 116 116 200 200 200 F F F 23 460-479 116 116 116 200 200 200 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 119, Max flush: 207 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.355612181Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.367353269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.367684496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.367727114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.367739963Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.367758011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"Crucible ec49da7d-0cca-4486-a415-a38cfe855152 has session id: 3620e364-ef57-4774-b39d-639a23fb6274","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368097848Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Upstairs opts: Upstairs UUID: ec49da7d-0cca-4486-a415-a38cfe855152, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368131786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Crucible stats registered with UUID: ec49da7d-0cca-4486-a415-a38cfe855152","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368147155Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:16.368160284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368333292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.36836728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368382559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368471533Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368489061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"ec49da7d-0cca-4486-a415-a38cfe855152 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.36851296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368525839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368540498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"1","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368554877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368568606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368582845Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368599944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"ds_connection connected from Ok(127.0.0.1:45700)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368636191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"ds_connection connected from Ok(127.0.0.1:37977)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.36865291Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"ds_connection connected from Ok(127.0.0.1:43150)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.368667949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.36939322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"1","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369422368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.36954071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369804632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369844589Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 205, 205, 196, 204, 207, 207, 200, 198, 205, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369911914Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[0]R generation[0..12]: [118, 114, 118, 118, 112, 118, 119, 119, 116, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369928903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369945372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 205, 205, 196, 204, 207, 207, 200, 198, 205, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369961071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[1]R generation[0..12]: [118, 114, 118, 118, 112, 118, 119, 119, 116, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.3699767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.369993339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 205, 205, 196, 204, 207, 207, 200, 198, 205, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370011897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[2]R generation[0..12]: [118, 114, 118, 118, 112, 118, 119, 119, 116, 114, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370027616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370043765Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370063784Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370078043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370091492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370105031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"":"downstairs","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370132269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"0","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370146618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370163267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"client":"2","":"downstairs","session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370177726Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370191115Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"ec49da7d-0cca-4486-a415-a38cfe855152 is now active with session: 3620e364-ef57-4774-b39d-639a23fb6274","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370205974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370220363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683,"session_id":"3620e364-ef57-4774-b39d-639a23fb6274"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370233982Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} {"msg":"Activated sub_volume ec49da7d-0cca-4486-a415-a38cfe855152","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:16.370245521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 267 len 16384 data: 7 10 10 11 02/30 Read block 253 len 12288 03/30 Write block 228 len 28672 data: 14 13 15 13 12 11 13 04/30 Read block 551 len 28672 05/30 Read block 397 len 12288 06/30 Write block 477 len 12288 data: 7 11 12 07/30 Read block 133 len 4096 08/30 Read block 253 len 40960 09/30 Read block 232 len 16384 10/30 Read block 353 len 8192 11/30 Write block 313 len 20480 data: 10 12 12 12 13 12/30 Read block 81 len 24576 13/30 Read block 181 len 24576 14/30 Read block 484 len 24576 15/30 Read block 284 len 16384 16/30 Write block 442 len 28672 data: 9 11 8 10 11 9 10 17/30 Write block 397 len 28672 data: 15 16 13 13 10 12 10 18/30 Flush 19/30 Write block 169 len 28672 data: 11 12 9 11 10 8 10 20/30 Read block 231 len 20480 21/30 Write block 466 len 32768 data: 10 11 11 11 13 12 11 9 22/30 Read block 181 len 20480 23/30 Write block 549 len 40960 data: 8 5 4 4 7 8 9 10 11 13 24/30 Write block 203 len 24576 data: 12 13 12 13 12 13 25/30 Write block 328 len 8192 data: 7 7 26/30 Write block 71 len 12288 data: 12 10 12 27/30 Read block 279 len 16384 28/30 Read block 196 len 40960 29/30 Read block 318 len 12288 30/30 Write block 367 len 20480 data: 11 8 8 10 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 29 29 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:16.382767176Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1683} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 201 201 201 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 118 118 118 205 205 205 F F F 3 060-079 118 118 118 205 205 205 T T F <--- 4 080-099 112 112 112 196 196 196 F F F 5 100-119 118 118 118 204 204 204 F F F 6 120-139 119 119 119 207 207 207 F F F 7 140-159 119 119 119 207 207 207 F F F 8 160-179 116 116 116 200 200 200 T T F <--- 9 180-199 114 114 114 198 198 198 F F F 10 200-219 118 118 118 205 205 205 T T F <--- 11 220-239 120 120 118 208 208 205 F F T <--- 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 119 208 208 207 F F T <--- 14 280-299 118 118 118 206 206 206 F F F 15 300-319 120 120 116 208 208 200 F F F <--- 16 320-339 119 119 119 207 207 207 T T F <--- 17 340-359 119 119 119 207 207 207 F F F 18 360-379 114 114 114 198 198 198 T T F <--- 19 380-399 120 120 118 208 208 204 F F F <--- 20 400-419 120 120 118 208 208 204 F F F <--- 21 420-439 111 111 111 195 195 195 F F F 22 440-459 120 120 116 208 208 200 F F F <--- 23 460-479 120 120 116 208 208 200 T T F <--- 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 115 115 115 199 199 199 T T F <--- 28 560-579 118 118 118 204 204 204 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 120, Max flush: 208 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 201 201 201 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 118 118 118 205 205 205 F F F 3 060-079 121 121 121 209 209 209 F F F 4 080-099 112 112 112 196 196 196 F F F 5 100-119 118 118 118 204 204 204 F F F 6 120-139 119 119 119 207 207 207 F F F 7 140-159 119 119 119 207 207 207 F F F 8 160-179 121 121 121 209 209 209 F F F 9 180-199 114 114 114 198 198 198 F F F 10 200-219 121 121 121 209 209 209 F F F 11 220-239 120 120 120 208 208 208 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 118 118 118 206 206 206 F F F 15 300-319 120 120 120 208 208 208 F F F 16 320-339 121 121 121 209 209 209 F F F 17 340-359 119 119 119 207 207 207 F F F 18 360-379 121 121 121 209 209 209 F F F 19 380-399 120 120 120 208 208 208 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 120 120 120 208 208 208 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 115 115 115 199 199 199 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 121, Max flush: 209 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.708108225Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:20.718897218Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.718953974Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.729361943Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.729658413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.72969602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.729708549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.729726648Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"Crucible 21fa09fa-266e-4236-a593-7de51e1b9f58 has session id: b5b466c2-7d5a-4bb5-8098-79213ff16338","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730049766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Upstairs opts: Upstairs UUID: 21fa09fa-266e-4236-a593-7de51e1b9f58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730120631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Crucible stats registered with UUID: 21fa09fa-266e-4236-a593-7de51e1b9f58","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.73013713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:20.730150579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730311778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730337726Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730352695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730475137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730492346Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"21fa09fa-266e-4236-a593-7de51e1b9f58 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730514754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730527453Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730541273Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730555272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730603138Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730628597Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730643306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"ds_connection connected from Ok(127.0.0.1:55957)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730658495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"ds_connection connected from Ok(127.0.0.1:52174)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730673463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"ds_connection connected from Ok(127.0.0.1:47306)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.730774397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.731410823Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.73145495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.73160307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.731768729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.731808786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 205, 209, 196, 204, 207, 207, 209, 198, 209, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732015642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[0]R generation[0..12]: [118, 114, 118, 121, 112, 118, 119, 119, 121, 114, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732032911Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732049409Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 205, 209, 196, 204, 207, 207, 209, 198, 209, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732065498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[1]R generation[0..12]: [118, 114, 118, 121, 112, 118, 119, 119, 121, 114, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732081137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732097256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 205, 209, 196, 204, 207, 207, 209, 198, 209, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732113305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[2]R generation[0..12]: [118, 114, 118, 121, 112, 118, 119, 119, 121, 114, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732128764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732144753Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:36:20.732158242Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732180361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.73219362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732207099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"":"downstairs","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732220578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732233117Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"0","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732246896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"1","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732263265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"client":"2","":"downstairs","session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732278264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732291663Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"21fa09fa-266e-4236-a593-7de51e1b9f58 is now active with session: b5b466c2-7d5a-4bb5-8098-79213ff16338","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732306752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732319551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691,"session_id":"b5b466c2-7d5a-4bb5-8098-79213ff16338"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.73233273Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} {"msg":"Activated sub_volume 21fa09fa-266e-4236-a593-7de51e1b9f58","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:20.732344039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 91 len 28672 data: 6 5 7 6 8 12 11 02/30 Write block 214 len 32768 data: 6 8 9 10 9 10 13 11 03/30 Read block 308 len 40960 04/30 Write block 191 len 4096 data: 7 05/30 Read block 431 len 40960 06/30 Read block 552 len 32768 07/30 Write block 233 len 12288 data: 12 14 14 08/30 Read block 514 len 32768 09/30 Flush 10/30 Write block 218 len 32768 data: 10 11 14 12 14 16 17 14 11/30 Write block 137 len 40960 data: 11 13 10 12 11 10 8 9 8 6 12/30 Read block 496 len 16384 13/30 Write block 52 len 28672 data: 9 7 6 6 8 8 8 14/30 Write block 9 len 24576 data: 14 15 13 15 12 11 15/30 Write block 129 len 4096 data: 8 16/30 Flush 17/30 Write block 102 len 24576 data: 10 9 7 8 8 5 18/30 Write block 512 len 32768 data: 8 6 8 10 11 13 14 12 19/30 Read block 420 len 16384 20/30 Write block 452 len 12288 data: 17 14 14 21/30 Read block 554 len 8192 22/30 Flush 23/30 Write block 232 len 4096 data: 13 24/30 Flush 25/30 Write block 89 len 8192 data: 5 7 26/30 Write block 178 len 36864 data: 8 9 11 12 13 11 10 9 8 27/30 Write block 537 len 8192 data: 8 7 28/30 Write block 380 len 16384 data: 13 10 10 12 29/30 Read block 468 len 32768 30/30 Read block 418 len 28672 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:20.744569514Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1691} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 122 122 201 211 211 F F F <--- 1 020-039 114 114 114 198 198 198 F F F 2 040-059 118 122 122 205 211 211 F F F <--- 3 060-079 121 121 121 209 209 209 F F F 4 080-099 112 122 122 196 210 210 T T T <--- 5 100-119 118 122 122 204 212 212 F F F <--- 6 120-139 119 122 122 207 211 211 F F F <--- 7 140-159 119 122 122 207 211 211 F F F <--- 8 160-179 121 121 121 209 209 209 F T T <--- 9 180-199 114 122 122 198 210 210 F T T <--- 10 200-219 121 122 122 209 211 211 F F F <--- 11 220-239 120 122 122 208 213 213 F F F <--- 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 118 118 118 206 206 206 F F F 15 300-319 120 120 120 208 208 208 F F F 16 320-339 121 121 121 209 209 209 F F F 17 340-359 119 119 119 207 207 207 F F F 18 360-379 121 121 121 209 209 209 F F F 19 380-399 120 120 120 208 208 208 F T T <--- 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 120 122 122 208 212 212 F F F <--- 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 114 122 122 197 212 212 F F F <--- 26 520-539 115 115 115 199 199 199 F T T <--- 27 540-559 121 121 121 209 209 209 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 122, Max flush: 213 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 211 211 211 F F F 1 020-039 114 114 114 198 198 198 F F F 2 040-059 122 122 122 211 211 211 F F F 3 060-079 121 121 121 209 209 209 F F F 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 122 122 122 211 211 211 F F F 7 140-159 122 122 122 211 211 211 F F F 8 160-179 123 123 123 214 214 214 F F F 9 180-199 123 123 123 214 214 214 F F F 10 200-219 122 122 122 211 211 211 F F F 11 220-239 122 122 122 213 213 213 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 118 118 118 206 206 206 F F F 15 300-319 120 120 120 208 208 208 F F F 16 320-339 121 121 121 209 209 209 F F F 17 340-359 119 119 119 207 207 207 F F F 18 360-379 121 121 121 209 209 209 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 122 122 122 212 212 212 F F F 26 520-539 123 123 123 214 214 214 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 118 118 118 204 204 204 F F F 29 580-599 111 111 111 195 195 195 F F F Max gen: 123, Max flush: 214 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.08225035Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.09424384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.094587257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.094623814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.094638703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.094656792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"Crucible 1ea9bfbf-2e37-475a-8c27-61f26be21ded has session id: 207124c8-ddaa-48d1-8807-97dcd5966226","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.094971061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Upstairs opts: Upstairs UUID: 1ea9bfbf-2e37-475a-8c27-61f26be21ded, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095018277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Crucible stats registered with UUID: 1ea9bfbf-2e37-475a-8c27-61f26be21ded","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095034326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:25.095047905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095223513Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095248412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095263021Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095383623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095401791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"1ea9bfbf-2e37-475a-8c27-61f26be21ded active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.09542391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095436879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095450928Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095467237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095481156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095495305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095509544Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"ds_connection connected from Ok(127.0.0.1:55406)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095543562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"ds_connection connected from Ok(127.0.0.1:35753)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095559451Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"ds_connection connected from Ok(127.0.0.1:38408)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.095574209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096347407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096367285Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096393174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096719401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096764608Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[0]R flush_numbers[0..12]: [211, 198, 211, 209, 214, 212, 211, 211, 214, 214, 211, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096801706Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[0]R generation[0..12]: [122, 114, 122, 121, 123, 122, 122, 122, 123, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096818954Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096838013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[1]R flush_numbers[0..12]: [211, 198, 211, 209, 214, 212, 211, 211, 214, 214, 211, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096853802Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[1]R generation[0..12]: [122, 114, 122, 121, 123, 122, 122, 122, 123, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096876131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096892769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[2]R flush_numbers[0..12]: [211, 198, 211, 209, 214, 212, 211, 211, 214, 214, 211, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096908368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[2]R generation[0..12]: [122, 114, 122, 121, 123, 122, 122, 122, 123, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096923917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096940166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096959195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096973584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.096986713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097000002Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"":"downstairs","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097014261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.09702707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"0","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097040989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"1","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097058908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"client":"2","":"downstairs","session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097073677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097087206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"1ea9bfbf-2e37-475a-8c27-61f26be21ded is now active with session: 207124c8-ddaa-48d1-8807-97dcd5966226","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097102635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097115314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699,"session_id":"207124c8-ddaa-48d1-8807-97dcd5966226"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097128713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} {"msg":"Activated sub_volume 1ea9bfbf-2e37-475a-8c27-61f26be21ded","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:25.097140162Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 202 len 4096 data: 12 03/30 Write block 200 len 32768 data: 11 12 13 13 14 13 14 13 04/30 Read block 219 len 4096 05/30 Write block 576 len 36864 data: 6 5 7 8 6 9 9 10 10 06/30 Write block 354 len 36864 data: 12 16 17 17 15 15 13 13 10 07/30 Read block 244 len 4096 08/30 Read block 527 len 20480 09/30 Read block 36 len 20480 10/30 Read block 506 len 28672 11/30 Write block 290 len 36864 data: 6 6 5 5 5 7 8 11 10 12/30 Write block 376 len 4096 data: 12 13/30 Write block 63 len 32768 data: 15 17 16 15 14 15 15 12 14/30 Write block 2 len 24576 data: 6 8 7 9 10 10 15/30 Read block 545 len 32768 16/30 Write block 296 len 8192 data: 9 12 17/30 Read block 315 len 12288 18/30 Write block 370 len 28672 data: 11 11 10 10 10 13 13 19/30 Read block 2 len 40960 20/30 Flush 21/30 Flush 22/30 Write block 6 len 16384 data: 11 11 11 15 23/30 Write block 168 len 28672 data: 11 12 13 10 12 11 9 24/30 Flush 25/30 Flush 26/30 Write block 70 len 20480 data: 13 13 11 13 9 27/30 Read block 175 len 28672 28/30 Read block 51 len 4096 29/30 Write block 33 len 16384 data: 11 11 11 11 30/30 Read block 411 len 20480 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 29 31 Done 29 27 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:25.109137933Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1699} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 122 218 218 211 F F F <--- 1 020-039 114 114 114 198 198 198 T T F <--- 2 040-059 122 122 122 211 211 211 F F F 3 060-079 124 124 121 216 216 209 T T F <--- 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 122 122 122 211 211 211 F F F 7 140-159 122 122 122 211 211 211 F F F 8 160-179 124 124 123 218 218 214 F F F <--- 9 180-199 123 123 123 214 214 214 F F F 10 200-219 124 124 122 216 216 211 F F F <--- 11 220-239 122 122 122 213 213 213 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 124 124 118 216 216 206 F F F <--- 15 300-319 120 120 120 208 208 208 F F F 16 320-339 121 121 121 209 209 209 F F F 17 340-359 124 124 119 216 216 207 F F F <--- 18 360-379 124 124 121 216 216 209 F F F <--- 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 122 122 122 212 212 212 F F F 26 520-539 123 123 123 214 214 214 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 124 124 118 216 216 204 F F F <--- 29 580-599 124 124 111 216 216 195 F F F <--- Max gen: 124, Max flush: 218 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 125 125 125 219 219 219 F F F 2 040-059 122 122 122 211 211 211 F F F 3 060-079 125 125 125 219 219 219 F F F 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 122 122 122 211 211 211 F F F 7 140-159 122 122 122 211 211 211 F F F 8 160-179 124 124 124 218 218 218 F F F 9 180-199 123 123 123 214 214 214 F F F 10 200-219 124 124 124 216 216 216 F F F 11 220-239 122 122 122 213 213 213 F F F 12 240-259 116 116 116 200 200 200 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 124 124 124 216 216 216 F F F 15 300-319 120 120 120 208 208 208 F F F 16 320-339 121 121 121 209 209 209 F F F 17 340-359 124 124 124 216 216 216 F F F 18 360-379 124 124 124 216 216 216 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 122 122 122 212 212 212 F F F 26 520-539 123 123 123 214 214 214 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 124 124 124 216 216 216 F F F 29 580-599 124 124 124 216 216 216 F F F Max gen: 125, Max flush: 219 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.424203474Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436058104Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436408281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436447548Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436459797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436477726Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"Crucible 15dacfd4-2a3e-4272-a4d0-5616ef9e772d has session id: 68fd2fc8-b26c-4786-8950-50cf481deca3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436802354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Upstairs opts: Upstairs UUID: 15dacfd4-2a3e-4272-a4d0-5616ef9e772d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436877148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Crucible stats registered with UUID: 15dacfd4-2a3e-4272-a4d0-5616ef9e772d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.436895937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:29.436909536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437080345Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437105253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437120232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437235314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437253203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"15dacfd4-2a3e-4272-a4d0-5616ef9e772d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437274011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.43728686Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.43730109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437314909Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437328588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437342857Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437356886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37033)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437376064Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57727)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437423961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65073)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.437451779Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.4381769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438214187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438231456Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438542895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438587542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[0]R flush_numbers[0..12]: [218, 219, 211, 219, 214, 212, 211, 211, 218, 214, 216, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438628269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[0]R generation[0..12]: [124, 125, 122, 125, 123, 122, 122, 122, 124, 123, 124, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438645518Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438662007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[1]R flush_numbers[0..12]: [218, 219, 211, 219, 214, 212, 211, 211, 218, 214, 216, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438677965Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[1]R generation[0..12]: [124, 125, 122, 125, 123, 122, 122, 122, 124, 123, 124, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438693944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438710463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[2]R flush_numbers[0..12]: [218, 219, 211, 219, 214, 212, 211, 211, 218, 214, 216, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438726492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[2]R generation[0..12]: [124, 125, 122, 125, 123, 122, 122, 122, 124, 123, 124, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438742391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.43876189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438781148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438795507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438808727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438822136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"":"downstairs","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438836555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438849574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"0","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438863333Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"1","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438879612Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"client":"2","":"downstairs","session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438893921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.43890756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"15dacfd4-2a3e-4272-a4d0-5616ef9e772d is now active with session: 68fd2fc8-b26c-4786-8950-50cf481deca3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438922479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438935088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707,"session_id":"68fd2fc8-b26c-4786-8950-50cf481deca3"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438948327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} {"msg":"Activated sub_volume 15dacfd4-2a3e-4272-a4d0-5616ef9e772d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:29.438959646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 230 len 16384 data: 16 14 14 13 02/30 Write block 179 len 4096 data: 10 03/30 Read block 578 len 36864 04/30 Write block 123 len 16384 data: 10 11 10 11 05/30 Read block 542 len 32768 06/30 Flush 07/30 Write block 239 len 20480 data: 8 7 8 12 11 08/30 Read block 172 len 28672 09/30 Read block 106 len 4096 10/30 Read block 19 len 12288 11/30 Write block 218 len 40960 data: 11 12 15 13 15 17 18 15 14 16 12/30 Write block 202 len 24576 data: 14 14 15 14 15 14 13/30 Write block 200 len 40960 data: 12 13 15 15 16 15 16 15 14 14 14/30 Write block 140 len 28672 data: 13 12 11 9 10 9 7 15/30 Write block 561 len 40960 data: 12 11 11 13 9 10 8 8 5 8 16/30 Read block 204 len 20480 17/30 Read block 532 len 40960 18/30 Flush 19/30 Write block 313 len 32768 data: 11 13 13 13 14 14 12 11 20/30 Write block 501 len 32768 data: 12 15 20 18 17 16 17 14 21/30 Read block 5 len 36864 22/30 Write block 335 len 16384 data: 11 12 11 9 23/30 Write block 176 len 24576 data: 9 9 9 11 12 13 24/30 Write block 361 len 12288 data: 14 11 8 25/30 Read block 354 len 4096 26/30 Read block 472 len 12288 27/30 Read block 21 len 16384 28/30 Read block 402 len 4096 29/30 Read block 564 len 40960 30/30 Write block 59 len 16384 data: 8 12 12 13 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Write 1 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:29.450628139Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1707} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 125 125 125 219 219 219 F F F 2 040-059 122 122 122 211 211 211 T T F <--- 3 060-079 125 125 125 219 219 219 T T F <--- 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 126 126 122 220 220 211 F F F <--- 7 140-159 126 126 122 221 221 211 F F F <--- 8 160-179 126 126 124 220 220 218 T T F <--- 9 180-199 123 123 123 214 214 214 T T F <--- 10 200-219 126 126 124 221 221 216 F F F <--- 11 220-239 126 126 122 221 221 213 F F F <--- 12 240-259 126 126 116 221 221 200 F F F <--- 13 260-279 120 120 120 208 208 208 F F F 14 280-299 124 124 124 216 216 216 F F F 15 300-319 120 120 120 208 208 208 T T F <--- 16 320-339 121 121 121 209 209 209 T T F <--- 17 340-359 124 124 124 216 216 216 F F F 18 360-379 124 124 124 216 216 216 T T F <--- 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 122 122 122 212 212 212 T T F <--- 26 520-539 123 123 123 214 214 214 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 126 126 124 221 221 216 F F F <--- 29 580-599 124 124 124 216 216 216 F F F Max gen: 126, Max flush: 221 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 125 125 125 219 219 219 F F F 2 040-059 127 127 127 222 222 222 F F F 3 060-079 127 127 127 222 222 222 F F F 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 126 126 126 220 220 220 F F F 7 140-159 126 126 126 221 221 221 F F F 8 160-179 127 127 127 222 222 222 F F F 9 180-199 127 127 127 222 222 222 F F F 10 200-219 126 126 126 221 221 221 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 120 120 120 208 208 208 F F F 14 280-299 124 124 124 216 216 216 F F F 15 300-319 127 127 127 222 222 222 F F F 16 320-339 127 127 127 222 222 222 F F F 17 340-359 124 124 124 216 216 216 F F F 18 360-379 127 127 127 222 222 222 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 127 127 127 222 222 222 F F F 26 520-539 123 123 123 214 214 214 F F F 27 540-559 121 121 121 209 209 209 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 124 124 124 216 216 216 F F F Max gen: 127, Max flush: 222 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.779619129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.791294341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.791621439Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.791664226Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.791676605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.791694794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"Crucible 207ec89c-d0ab-4b37-94ee-d836f4df7e32 has session id: cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792033121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Upstairs opts: Upstairs UUID: 207ec89c-d0ab-4b37-94ee-d836f4df7e32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792066919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Crucible stats registered with UUID: 207ec89c-d0ab-4b37-94ee-d836f4df7e32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792082567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:33.792096087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792288783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792315642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792330441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792450772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792469011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"207ec89c-d0ab-4b37-94ee-d836f4df7e32 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792538306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792554715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792569014Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792583193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792597062Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792611251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.79262553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"ds_connection connected from Ok(127.0.0.1:39608)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792722824Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"ds_connection connected from Ok(127.0.0.1:42449)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792740783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"ds_connection connected from Ok(127.0.0.1:39367)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.792755641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793433125Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793472192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793518619Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793823028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793860576Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[0]R flush_numbers[0..12]: [218, 219, 222, 222, 214, 212, 220, 221, 222, 222, 221, 221]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793924951Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[0]R generation[0..12]: [124, 125, 127, 127, 123, 122, 126, 126, 127, 127, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.79394631Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793963549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[1]R flush_numbers[0..12]: [218, 219, 222, 222, 214, 212, 220, 221, 222, 222, 221, 221]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.793991707Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[1]R generation[0..12]: [124, 125, 127, 127, 123, 122, 126, 126, 127, 127, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794007876Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794024894Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[2]R flush_numbers[0..12]: [218, 219, 222, 222, 214, 212, 220, 221, 222, 222, 221, 221]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794041043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[2]R generation[0..12]: [124, 125, 127, 127, 123, 122, 126, 126, 127, 127, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794056792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2026-03-16T21:36:33.794079331Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.79409327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794107859Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794121568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794135847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"":"downstairs","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794150096Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794162965Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"0","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794182304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"1","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794196583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"client":"2","":"downstairs","session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794211012Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794224651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"207ec89c-d0ab-4b37-94ee-d836f4df7e32 is now active with session: cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.79423967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794252249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715,"session_id":"cbfb77e1-9ce5-40ce-82b4-5a96ba1b6c49"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794265738Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} {"msg":"Activated sub_volume 207ec89c-d0ab-4b37-94ee-d836f4df7e32","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:33.794277207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 25 len 20480 data: 9 9 10 10 12 02/30 Read block 148 len 20480 03/30 Write block 262 len 40960 data: 8 9 9 8 8 8 11 11 12 12 04/30 Read block 261 len 24576 05/30 Write block 239 len 32768 data: 9 8 9 13 12 10 11 11 06/30 Read block 350 len 36864 07/30 Write block 149 len 28672 data: 9 8 9 9 8 6 7 08/30 Write block 30 len 20480 data: 13 12 14 12 12 09/30 Flush 10/30 Write block 133 len 8192 data: 8 10 11/30 Write block 241 len 16384 data: 10 14 13 11 12/30 Write block 67 len 8192 data: 15 16 13/30 Read block 133 len 24576 14/30 Read block 206 len 40960 15/30 Read block 492 len 16384 16/30 Read block 466 len 40960 17/30 Write block 220 len 16384 data: 16 14 16 18 18/30 Read block 132 len 8192 19/30 Read block 304 len 32768 20/30 Read block 418 len 28672 21/30 Read block 147 len 8192 22/30 Write block 558 len 8192 data: 14 11 23/30 Read block 178 len 24576 24/30 Read block 240 len 28672 25/30 Flush 26/30 Write block 460 len 16384 data: 14 9 8 8 27/30 Write block 361 len 12288 data: 15 12 9 28/30 Write block 570 len 24576 data: 9 9 8 6 9 8 29/30 Read block 237 len 24576 30/30 Write block 505 len 16384 data: 18 17 18 15 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 26 28 Done 29 29 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:33.806063752Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1715} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 125 223 223 219 F F T <--- 2 040-059 127 127 127 222 222 222 F F F 3 060-079 128 128 127 224 224 222 F F F <--- 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 126 224 224 220 F F F <--- 7 140-159 128 128 126 223 223 221 F F F <--- 8 160-179 127 127 127 222 222 222 F F F 9 180-199 127 127 127 222 222 222 F F F 10 200-219 126 126 126 221 221 221 F F F 11 220-239 128 128 126 224 224 221 F F F <--- 12 240-259 128 128 126 224 224 221 F F F <--- 13 260-279 128 128 120 223 223 208 F F T <--- 14 280-299 124 124 124 216 216 216 F F F 15 300-319 127 127 127 222 222 222 F F F 16 320-339 127 127 127 222 222 222 F F F 17 340-359 124 124 124 216 216 216 F F F 18 360-379 127 127 127 222 222 222 T T F <--- 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 121 121 121 209 209 209 T T F <--- 24 480-499 116 116 116 200 200 200 F F F 25 500-519 127 127 127 222 222 222 T T F <--- 26 520-539 123 123 123 214 214 214 F F F 27 540-559 128 128 121 224 224 209 F F F <--- 28 560-579 126 126 126 221 221 221 T T F <--- 29 580-599 124 124 124 216 216 216 F F F Max gen: 128, Max flush: 224 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 127 127 127 222 222 222 F F F 3 060-079 128 128 128 224 224 224 F F F 4 080-099 123 123 123 214 214 214 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 128 224 224 224 F F F 7 140-159 128 128 128 223 223 223 F F F 8 160-179 127 127 127 222 222 222 F F F 9 180-199 127 127 127 222 222 222 F F F 10 200-219 126 126 126 221 221 221 F F F 11 220-239 128 128 128 224 224 224 F F F 12 240-259 128 128 128 224 224 224 F F F 13 260-279 128 128 128 223 223 223 F F F 14 280-299 124 124 124 216 216 216 F F F 15 300-319 127 127 127 222 222 222 F F F 16 320-339 127 127 127 222 222 222 F F F 17 340-359 124 124 124 216 216 216 F F F 18 360-379 129 129 129 225 225 225 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 F F F 24 480-499 116 116 116 200 200 200 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 123 123 123 214 214 214 F F F 27 540-559 128 128 128 224 224 224 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 124 124 124 216 216 216 F F F Max gen: 129, Max flush: 225 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.127829146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:38.138534044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.13859543Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.148872988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149175047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149213404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149225654Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149243482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"Crucible 144dc9d4-5bbc-4778-a10b-915b17c3318c has session id: 5ef03bd6-dcd4-419c-a764-87f9129a6deb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149556321Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Upstairs opts: Upstairs UUID: 144dc9d4-5bbc-4778-a10b-915b17c3318c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149614457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Crucible stats registered with UUID: 144dc9d4-5bbc-4778-a10b-915b17c3318c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149631246Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:38.149644875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149828912Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149853531Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.14986832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149969833Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.149988461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"144dc9d4-5bbc-4778-a10b-915b17c3318c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.15000942Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150022039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150036368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150050427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150064256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150078635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150114763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62955)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150165819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56266)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150191898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48316)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150207756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.150909649Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151000182Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151055839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151279683Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151334509Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[0]R flush_numbers[0..12]: [218, 223, 222, 224, 214, 212, 224, 223, 222, 222, 221, 224]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151459061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[0]R generation[0..12]: [124, 128, 127, 128, 123, 122, 128, 128, 127, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.15147648Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151493519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[1]R flush_numbers[0..12]: [218, 223, 222, 224, 214, 212, 224, 223, 222, 222, 221, 224]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151509338Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[1]R generation[0..12]: [124, 128, 127, 128, 123, 122, 128, 128, 127, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151524796Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151544445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[2]R flush_numbers[0..12]: [218, 223, 222, 224, 214, 212, 224, 223, 222, 222, 221, 224]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151560614Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[2]R generation[0..12]: [124, 128, 127, 128, 123, 122, 128, 128, 127, 127, 126, 128]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151576253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151592322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151605701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.15162531Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151639259Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151652528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"":"downstairs","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151666497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151679436Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"0","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151693475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"1","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151707414Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"client":"2","":"downstairs","session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151721693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:36:38.151734722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"144dc9d4-5bbc-4778-a10b-915b17c3318c is now active with session: 5ef03bd6-dcd4-419c-a764-87f9129a6deb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151751691Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.1517661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723,"session_id":"5ef03bd6-dcd4-419c-a764-87f9129a6deb"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151779799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} {"msg":"Activated sub_volume 144dc9d4-5bbc-4778-a10b-915b17c3318c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:38.151791388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 28672 data: 7 8 7 7 12 13 12 02/30 Read block 26 len 40960 03/30 Read block 456 len 28672 04/30 Read block 288 len 32768 05/30 Flush 06/30 Write block 74 len 36864 data: 10 8 8 8 6 7 6 8 7 07/30 Read block 373 len 20480 08/30 Write block 241 len 12288 data: 11 15 14 09/30 Read block 308 len 40960 10/30 Write block 261 len 12288 data: 10 9 10 11/30 Read block 334 len 4096 12/30 Write block 219 len 8192 data: 13 17 13/30 Read block 376 len 4096 14/30 Write block 186 len 12288 data: 9 6 5 15/30 Write block 249 len 12288 data: 10 9 8 16/30 Write block 307 len 12288 data: 8 8 8 17/30 Write block 520 len 12288 data: 12 10 9 18/30 Read block 359 len 24576 19/30 Read block 297 len 8192 20/30 Read block 544 len 12288 21/30 Read block 518 len 32768 22/30 Flush 23/30 Write block 480 len 8192 data: 14 13 24/30 Read block 478 len 20480 25/30 Write block 176 len 36864 data: 10 10 10 12 13 14 14 12 11 26/30 Write block 313 len 4096 data: 12 27/30 Write block 359 len 24576 data: 16 14 16 13 10 8 28/30 Write block 355 len 8192 data: 17 18 29/30 Read block 285 len 32768 30/30 Read block 93 len 40960 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:38.163715353Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1723} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 127 127 127 222 222 222 F F F 3 060-079 128 130 130 224 227 227 F F F <--- 4 080-099 123 130 130 214 227 227 F F F <--- 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 128 224 224 224 F F F 7 140-159 128 128 128 223 223 223 F F F 8 160-179 127 127 127 222 222 222 F T T <--- 9 180-199 127 130 130 222 227 227 F T T <--- 10 200-219 126 130 130 221 227 227 F F F <--- 11 220-239 128 130 130 224 227 227 F F F <--- 12 240-259 128 130 130 224 227 227 F F F <--- 13 260-279 128 130 130 223 227 227 F F F <--- 14 280-299 124 124 124 216 216 216 F F F 15 300-319 127 130 130 222 227 227 F T T <--- 16 320-339 127 130 130 222 226 226 T F F <--- 17 340-359 124 124 124 216 216 216 F T T <--- 18 360-379 129 129 129 225 225 225 F T T <--- 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 F F F 24 480-499 116 116 116 200 200 200 F T T <--- 25 500-519 129 129 129 225 225 225 F F F 26 520-539 123 130 130 214 227 227 F F F <--- 27 540-559 128 128 128 224 224 224 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 124 124 124 216 216 216 F F F Max gen: 130, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 127 127 127 222 222 222 F F F 3 060-079 130 130 130 227 227 227 F F F 4 080-099 130 130 130 227 227 227 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 128 224 224 224 F F F 7 140-159 128 128 128 223 223 223 F F F 8 160-179 131 131 131 228 228 228 F F F 9 180-199 131 131 131 228 228 228 F F F 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F F F 12 240-259 130 130 130 227 227 227 F F F 13 260-279 130 130 130 227 227 227 F F F 14 280-299 124 124 124 216 216 216 F F F 15 300-319 131 131 131 228 228 228 F F F 16 320-339 130 130 130 226 226 226 F F F 17 340-359 131 131 131 228 228 228 F F F 18 360-379 131 131 131 228 228 228 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 F F F 24 480-499 131 131 131 228 228 228 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 128 128 128 224 224 224 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 124 124 124 216 216 216 F F F Max gen: 131, Max flush: 228 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.502203925Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:42.512877645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.512930132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523220829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523525238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523560045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523572315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523590073Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"Crucible 7dd78edf-25e2-4548-8a0c-2613b41216df has session id: 741bafe7-f7df-4030-af2b-0675eb6717a8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523915081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Upstairs opts: Upstairs UUID: 7dd78edf-25e2-4548-8a0c-2613b41216df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523948999Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Crucible stats registered with UUID: 7dd78edf-25e2-4548-8a0c-2613b41216df","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.523963658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:42.523976957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524150355Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524176223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524190902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524302635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524323183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"7dd78edf-25e2-4548-8a0c-2613b41216df active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524340372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524353231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.52436735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524381829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524414877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524459044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524483562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52045)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524499221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56304)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.52451414Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:44180)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.524577026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525171135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525237601Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525359173Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525523711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525560379Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[0]R flush_numbers[0..12]: [218, 223, 222, 227, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525741866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[0]R generation[0..12]: [124, 128, 127, 130, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525759475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525776094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[1]R flush_numbers[0..12]: [218, 223, 222, 227, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525791933Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[1]R generation[0..12]: [124, 128, 127, 130, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525807852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525824221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[2]R flush_numbers[0..12]: [218, 223, 222, 227, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.52583966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[2]R generation[0..12]: [124, 128, 127, 130, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525855429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525871388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525892626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525906945Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525920314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525933663Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"":"downstairs","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525947252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.525960281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"0","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.52597497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"1","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.52598867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"client":"2","":"downstairs","session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-16T21:36:42.526002629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.526018717Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"7dd78edf-25e2-4548-8a0c-2613b41216df is now active with session: 741bafe7-f7df-4030-af2b-0675eb6717a8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.526034076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.526046856Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731,"session_id":"741bafe7-f7df-4030-af2b-0675eb6717a8"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.526060575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} {"msg":"Activated sub_volume 7dd78edf-25e2-4548-8a0c-2613b41216df","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:42.526072094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 40960 data: 12 11 11 11 14 14 13 14 14 14 02/30 Read block 120 len 32768 03/30 Flush 04/30 Write block 56 len 16384 data: 9 9 9 9 05/30 Write block 586 len 28672 data: 8 5 4 5 4 4 5 06/30 Read block 108 len 28672 07/30 Read block 89 len 28672 08/30 Read block 40 len 20480 09/30 Read block 175 len 4096 10/30 Flush 11/30 Write block 68 len 36864 data: 17 16 14 14 12 14 11 9 9 12/30 Read block 499 len 40960 13/30 Read block 239 len 36864 14/30 Read block 356 len 24576 15/30 Read block 464 len 20480 16/30 Write block 305 len 20480 data: 6 7 9 9 9 17/30 Read block 221 len 8192 18/30 Read block 87 len 20480 19/30 Flush 20/30 Write block 67 len 32768 data: 16 18 17 15 15 13 15 12 21/30 Read block 45 len 12288 22/30 Write block 290 len 28672 data: 7 7 6 6 6 8 10 23/30 Write block 341 len 4096 data: 9 24/30 Write block 423 len 12288 data: 9 10 10 25/30 Read block 305 len 32768 26/30 Write block 312 len 32768 data: 8 13 14 14 14 15 15 13 27/30 Read block 40 len 20480 28/30 Read block 355 len 4096 29/30 Read block 90 len 12288 30/30 Read block 598 len 4096 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 1 22 Done 6 27 26 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:42.539210466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1731} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 127 132 132 222 230 230 T F F <--- 3 060-079 130 132 132 227 231 231 F T T <--- 4 080-099 130 130 130 227 227 227 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 128 224 224 224 F F F 7 140-159 128 128 128 223 223 223 F F F 8 160-179 131 131 131 228 228 228 F F F 9 180-199 131 131 131 228 228 228 F F F 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F F F 12 240-259 130 130 130 227 227 227 F F F 13 260-279 130 130 130 227 227 227 F F F 14 280-299 124 124 124 216 216 216 F T T <--- 15 300-319 131 132 132 228 231 231 F T T <--- 16 320-339 130 130 130 226 226 226 F F F 17 340-359 131 131 131 228 228 228 F T T <--- 18 360-379 132 132 132 229 229 229 F F F 19 380-399 132 132 132 229 229 229 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 111 111 111 195 195 195 F T T <--- 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 F F F 24 480-499 131 131 131 228 228 228 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 128 128 128 224 224 224 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 124 132 132 216 230 230 T F F <--- Max gen: 132, Max flush: 231 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 218 218 218 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 132 132 132 230 230 230 F F F 3 060-079 133 133 133 232 232 232 F F F 4 080-099 130 130 130 227 227 227 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 128 128 128 224 224 224 F F F 7 140-159 128 128 128 223 223 223 F F F 8 160-179 131 131 131 228 228 228 F F F 9 180-199 131 131 131 228 228 228 F F F 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F F F 12 240-259 130 130 130 227 227 227 F F F 13 260-279 130 130 130 227 227 227 F F F 14 280-299 133 133 133 232 232 232 F F F 15 300-319 133 133 133 232 232 232 F F F 16 320-339 130 130 130 226 226 226 F F F 17 340-359 133 133 133 232 232 232 F F F 18 360-379 132 132 132 229 229 229 F F F 19 380-399 132 132 132 229 229 229 F F F 20 400-419 120 120 120 208 208 208 F F F 21 420-439 133 133 133 232 232 232 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 F F F 24 480-499 131 131 131 228 228 228 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 128 128 128 224 224 224 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 132 132 132 230 230 230 F F F Max gen: 133, Max flush: 232 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.857871602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.869720092Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.8700389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870076288Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870091677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870109715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"Crucible f7c5f5ca-9508-4f96-a040-5024c2d9abb5 has session id: f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870446872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Upstairs opts: Upstairs UUID: f7c5f5ca-9508-4f96-a040-5024c2d9abb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870492739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Crucible stats registered with UUID: f7c5f5ca-9508-4f96-a040-5024c2d9abb5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870508598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:46.870522087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870691886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870717174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870732043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870837256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870856174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"f7c5f5ca-9508-4f96-a040-5024c2d9abb5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870875433Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870888342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870902371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.8709165Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870932939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870955188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870969507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55062)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.870994555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58804)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.871145185Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40716)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.871180712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.871746694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.871782461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.8717994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872102029Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872147306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[0]R flush_numbers[0..12]: [218, 223, 230, 232, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872207792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[0]R generation[0..12]: [124, 128, 132, 133, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872225001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.87224137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[1]R flush_numbers[0..12]: [218, 223, 230, 232, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872260339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[1]R generation[0..12]: [124, 128, 132, 133, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872277067Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872293486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[2]R flush_numbers[0..12]: [218, 223, 230, 232, 227, 212, 224, 223, 228, 228, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872309085Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[2]R generation[0..12]: [124, 128, 132, 133, 130, 122, 128, 128, 131, 131, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872324114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872340513Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Max found gen is 134"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872359812Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872374371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.87238777Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872401189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"":"downstairs","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872415268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872427937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"0","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872441946Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"1","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872459085Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"client":"2","":"downstairs","session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872473684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872489153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"f7c5f5ca-9508-4f96-a040-5024c2d9abb5 is now active with session: f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872504272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872517111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739,"session_id":"f3cf8b7b-f19b-4304-bdd7-fa22c3c77f7e"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.87253027Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} {"msg":"Activated sub_volume f7c5f5ca-9508-4f96-a040-5024c2d9abb5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:46.872541939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 424 len 12288 data: 11 11 13 02/30 Write block 137 len 24576 data: 12 14 11 14 13 12 03/30 Read block 42 len 4096 04/30 Read block 356 len 28672 05/30 Write block 371 len 20480 data: 13 12 12 12 15 06/30 Read block 510 len 28672 07/30 Read block 313 len 16384 08/30 Read block 549 len 8192 09/30 Write block 540 len 12288 data: 8 9 10 10/30 Write block 266 len 20480 data: 9 9 12 12 13 11/30 Write block 329 len 28672 data: 8 7 8 9 8 8 13 12/30 Read block 2 len 12288 13/30 Write block 127 len 28672 data: 10 9 9 8 7 8 9 14/30 Write block 185 len 4096 data: 10 15/30 Flush 16/30 Write block 13 len 28672 data: 13 12 11 9 12 11 11 17/30 Read block 313 len 28672 18/30 Read block 322 len 20480 19/30 Read block 456 len 8192 20/30 Read block 331 len 36864 21/30 Read block 430 len 40960 22/30 Write block 383 len 24576 data: 13 13 13 15 15 14 23/30 Read block 332 len 24576 24/30 Read block 296 len 16384 25/30 Write block 81 len 4096 data: 9 26/30 Flush 27/30 Write block 462 len 12288 data: 9 9 8 28/30 Read block 77 len 28672 29/30 Write block 42 len 32768 data: 10 9 8 7 8 8 9 9 30/30 Write block 415 len 20480 data: 8 9 10 8 9 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:46.884368501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1739} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 124 134 234 218 234 F F F <--- 1 020-039 128 128 128 223 223 223 F F F 2 040-059 132 132 132 230 230 230 T F T <--- 3 060-079 133 133 133 232 232 232 F F F 4 080-099 134 130 134 234 227 234 F F F <--- 5 100-119 122 122 122 212 212 212 F F F 6 120-139 134 128 134 233 224 233 F T F <--- 7 140-159 134 128 134 233 223 233 F T F <--- 8 160-179 131 131 131 228 228 228 F F F 9 180-199 134 131 134 233 228 233 F F F <--- 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F F F 12 240-259 130 130 130 227 227 227 F F F 13 260-279 134 130 134 233 227 233 F F F <--- 14 280-299 133 133 133 232 232 232 F F F 15 300-319 133 133 133 232 232 232 F F F 16 320-339 134 130 134 233 226 233 F F F <--- 17 340-359 133 133 133 232 232 232 F F F 18 360-379 134 132 134 233 229 233 F F F <--- 19 380-399 134 132 134 234 229 234 F F F <--- 20 400-419 120 120 120 208 208 208 T F T <--- 21 420-439 134 133 134 233 232 233 F T F <--- 22 440-459 122 122 122 212 212 212 F F F 23 460-479 129 129 129 225 225 225 T F T <--- 24 480-499 131 131 131 228 228 228 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 134 128 134 233 224 233 F F F <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 132 132 132 230 230 230 F F F Max gen: 134, Max flush: 234 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 234 234 234 F F F 1 020-039 128 128 128 223 223 223 F F F 2 040-059 135 135 135 235 235 235 F F F 3 060-079 133 133 133 232 232 232 F F F 4 080-099 134 134 134 234 234 234 F F F 5 100-119 122 122 122 212 212 212 F F F 6 120-139 134 134 134 233 233 233 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 131 131 131 228 228 228 F F F 9 180-199 134 134 134 233 233 233 F F F 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F F F 12 240-259 130 130 130 227 227 227 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 133 133 133 232 232 232 F F F 15 300-319 133 133 133 232 232 232 F F F 16 320-339 134 134 134 233 233 233 F F F 17 340-359 133 133 133 232 232 232 F F F 18 360-379 134 134 134 233 233 233 F F F 19 380-399 134 134 134 234 234 234 F F F 20 400-419 135 135 135 235 235 235 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 131 131 131 228 228 228 F F F 25 500-519 129 129 129 225 225 225 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 134 134 134 233 233 233 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 132 132 132 230 230 230 F F F Max gen: 135, Max flush: 235 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.212043441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:51.222802806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.222868632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.233328797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.233622097Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.233661704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.233673773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.233691602Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"Crucible 179112d4-c2b7-485c-bd6c-0eb5a03bf131 has session id: d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234047188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Upstairs opts: Upstairs UUID: 179112d4-c2b7-485c-bd6c-0eb5a03bf131, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234077676Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Crucible stats registered with UUID: 179112d4-c2b7-485c-bd6c-0eb5a03bf131","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234095304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:51.234108694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234285151Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.23431067Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234325739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234461189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234479198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"179112d4-c2b7-485c-bd6c-0eb5a03bf131 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234496307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234509446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234523705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234537574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234551283Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234565502Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234579591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"ds_connection connected from Ok(127.0.0.1:53237)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.23459635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"ds_connection connected from Ok(127.0.0.1:47645)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234611449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"ds_connection connected from Ok(127.0.0.1:50572)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.234731981Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235382816Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235410485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235434973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.23577722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235826386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[0]R flush_numbers[0..12]: [234, 223, 235, 232, 234, 212, 233, 233, 228, 233, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235890262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[0]R generation[0..12]: [134, 128, 135, 133, 134, 122, 134, 134, 131, 134, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235907611Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235924339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[1]R flush_numbers[0..12]: [234, 223, 235, 232, 234, 212, 233, 233, 228, 233, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235940178Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[1]R generation[0..12]: [134, 128, 135, 133, 134, 122, 134, 134, 131, 134, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235955797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235971996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[2]R flush_numbers[0..12]: [234, 223, 235, 232, 234, 212, 233, 233, 228, 233, 227, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.235987905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[2]R generation[0..12]: [134, 128, 135, 133, 134, 122, 134, 134, 131, 134, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236003764Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236023013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236037872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-16T21:36:51.236052171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236070839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236084708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"":"downstairs","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236099228Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236112087Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"0","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236126206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"1","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236140415Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"client":"2","":"downstairs","session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236154804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:36:51.236168313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"179112d4-c2b7-485c-bd6c-0eb5a03bf131 is now active with session: d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236186122Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236199071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747,"session_id":"d7c8f1bd-e1ac-4b2b-98ec-0da35a3cf827"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.23621306Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} {"msg":"Activated sub_volume 179112d4-c2b7-485c-bd6c-0eb5a03bf131","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:51.236224789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 24576 data: 9 9 7 8 10 10 02/30 Write block 124 len 16384 data: 12 11 12 11 03/30 Read block 546 len 36864 04/30 Flush 05/30 Write block 35 len 28672 data: 12 12 11 15 15 13 10 06/30 Write block 118 len 4096 data: 12 07/30 Read block 111 len 40960 08/30 Write block 179 len 12288 data: 13 14 15 09/30 Write block 489 len 24576 data: 14 10 11 10 11 13 10/30 Write block 321 len 16384 data: 11 12 10 10 11/30 Write block 550 len 28672 data: 6 5 5 8 9 10 11 12/30 Write block 361 len 40960 data: 17 14 11 9 6 11 12 9 9 12 13/30 Read block 318 len 40960 14/30 Read block 484 len 32768 15/30 Write block 246 len 12288 data: 12 8 10 16/30 Read block 531 len 32768 17/30 Write block 84 len 12288 data: 7 6 5 18/30 Write block 498 len 40960 data: 11 12 11 13 16 21 19 19 18 19 19/30 Read block 203 len 8192 20/30 Write block 35 len 36864 data: 13 13 12 16 16 14 11 11 10 21/30 Write block 227 len 32768 data: 17 15 14 17 15 15 14 15 22/30 Read block 489 len 16384 23/30 Read block 445 len 40960 24/30 Write block 63 len 40960 data: 16 18 17 16 17 19 18 16 16 14 25/30 Write block 402 len 36864 data: 13 11 11 11 12 14 13 15 12 26/30 Write block 412 len 16384 data: 13 11 11 9 27/30 Write block 407 len 8192 data: 15 14 28/30 Write block 287 len 32768 data: 9 10 8 8 8 7 7 7 29/30 Read block 452 len 40960 30/30 Write block 408 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 7 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 3 25 25 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:51.248336781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1747} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 234 234 234 F F F 1 020-039 128 128 128 223 223 223 T T T 2 040-059 135 135 135 235 235 235 T T T 3 060-079 133 133 133 232 232 232 F T T <--- 4 080-099 134 134 134 234 234 234 F T T <--- 5 100-119 122 122 122 212 212 212 T T T 6 120-139 136 136 136 236 236 236 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 131 131 131 228 228 228 F T T <--- 9 180-199 134 134 134 233 233 233 F T T <--- 10 200-219 130 130 130 227 227 227 F F F 11 220-239 130 130 130 227 227 227 F T T <--- 12 240-259 130 130 130 227 227 227 F T T <--- 13 260-279 134 134 134 233 233 233 F F F 14 280-299 133 133 133 232 232 232 F T T <--- 15 300-319 136 136 136 236 236 236 F F F 16 320-339 134 134 134 233 233 233 F T T <--- 17 340-359 133 133 133 232 232 232 F F F 18 360-379 134 134 134 233 233 233 F T T <--- 19 380-399 134 134 134 234 234 234 F F F 20 400-419 135 135 135 235 235 235 F T T <--- 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 131 131 131 228 228 228 F T T <--- 25 500-519 129 129 129 225 225 225 F T T <--- 26 520-539 130 130 130 227 227 227 F F F 27 540-559 134 134 134 233 233 233 F T T <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 132 132 132 230 230 230 F F F Max gen: 136, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 37a38,39 > 13, > 13, 39c41,43 < 12, --- > 16, > 16, > 14, 41,43c45 < 15, < 15, < 13, --- > 11, 45,46d46 < 10, < 9, diff found after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 234 234 234 F F F 1 020-039 137 137 137 237 237 237 F F F 2 040-059 137 137 137 237 237 237 F F F 3 060-079 137 137 137 237 237 237 F F F 4 080-099 137 137 137 237 237 237 F F F 5 100-119 137 137 137 237 237 237 F F F 6 120-139 136 136 136 236 236 236 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 130 130 130 227 227 227 F F F 11 220-239 137 137 137 237 237 237 F F F 12 240-259 137 137 137 237 237 237 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 137 137 137 237 237 237 F F F 15 300-319 136 136 136 236 236 236 F F F 16 320-339 137 137 137 237 237 237 F F F 17 340-359 133 133 133 232 232 232 F F F 18 360-379 137 137 137 237 237 237 F F F 19 380-399 134 134 134 234 234 234 F F F 20 400-419 137 137 137 237 237 237 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 137 137 137 237 237 237 F F F 25 500-519 137 137 137 237 237 237 F F F 26 520-539 130 130 130 227 227 227 F F F 27 540-559 137 137 137 237 237 237 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 132 132 132 230 230 230 F F F Max gen: 137, Max flush: 237 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.607952489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.619687557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620065481Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620105419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620117578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620135217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"Crucible de6936c3-af16-4842-88ac-7adc55540176 has session id: eeb16cb8-8231-41dc-9165-2b2a618613ac","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620451775Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Upstairs opts: Upstairs UUID: de6936c3-af16-4842-88ac-7adc55540176, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620497172Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Crucible stats registered with UUID: de6936c3-af16-4842-88ac-7adc55540176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620512801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:55.62052618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620710357Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620746005Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620761664Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620864837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620881885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"de6936c3-af16-4842-88ac-7adc55540176 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620899384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620912033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620926312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620940312Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620954311Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.62096934Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.620983559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:64374)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.621045094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:51353)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.62110341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:60567)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.621168966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.62184572Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.621903446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.621956412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622236753Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622265651Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[0]R flush_numbers[0..12]: [234, 237, 237, 237, 237, 237, 236, 233, 237, 237, 227, 237]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622332166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[0]R generation[0..12]: [134, 137, 137, 137, 137, 137, 136, 134, 137, 137, 130, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622349975Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622366454Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[1]R flush_numbers[0..12]: [234, 237, 237, 237, 237, 237, 236, 233, 237, 237, 227, 237]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622382603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[1]R generation[0..12]: [134, 137, 137, 137, 137, 137, 136, 134, 137, 137, 130, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622398162Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622414261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[2]R flush_numbers[0..12]: [234, 237, 237, 237, 237, 237, 236, 233, 237, 237, 227, 237]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.6224301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[2]R generation[0..12]: [134, 137, 137, 137, 137, 137, 136, 134, 137, 137, 130, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622445979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622462378Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622488476Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622502705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622516064Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622529303Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"":"downstairs","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622545272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622558011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"0","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.62257204Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"clientDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"1","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622589219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"client":"2","":"downstairs","session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622603448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622616737Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"de6936c3-af16-4842-88ac-7adc55540176 is now active with session: eeb16cb8-8231-41dc-9165-2b2a618613ac","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622631486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622643755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758,"session_id":"eeb16cb8-8231-41dc-9165-2b2a618613ac"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622656884Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} {"msg":"Activated sub_volume de6936c3-af16-4842-88ac-7adc55540176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:55.622668173Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 40960 data: 9 8 9 10 9 9 14 14 13 10 02/30 Read block 483 len 20480 03/30 Flush 04/30 Write block 541 len 8192 data: 10 11 05/30 Write block 500 len 24576 data: 12 14 17 22 20 20 06/30 Write block 581 len 12288 data: 10 10 11 07/30 Read block 217 len 12288 08/30 Read block 9 len 40960 09/30 Read block 215 len 24576 10/30 Read block 189 len 20480 11/30 Read block 488 len 28672 12/30 Read block 538 len 8192 13/30 Read block 372 len 32768 14/30 Read block 269 len 40960 15/30 Flush 16/30 Write block 473 len 24576 data: 10 8 6 6 8 12 17/30 Read block 333 len 20480 18/30 Flush 19/30 Write block 533 len 28672 data: 10 10 7 7 9 8 7 20/30 Read block 215 len 16384 21/30 Write block 243 len 16384 data: 15 12 12 13 22/30 Read block 97 len 32768 23/30 Read block 200 len 24576 24/30 Write block 364 len 36864 data: 10 7 12 13 10 10 13 14 13 25/30 Read block 488 len 20480 26/30 Read block 1 len 40960 27/30 Write block 205 len 24576 data: 16 17 16 15 15 13 28/30 Write block 331 len 4096 data: 10 29/30 Write block 225 len 36864 data: 16 15 18 16 15 18 16 16 15 30/30 Write block 14 len 24576 data: 13 12 10 13 12 12 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 2 Done Done Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 25 1 25 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1002)) Some(JobId(1017)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:55.635123332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1758} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 234 234 234 T F T <--- 1 020-039 137 137 137 237 237 237 F F F 2 040-059 137 137 137 237 237 237 F F F 3 060-079 137 137 137 237 237 237 F F F 4 080-099 137 137 137 237 237 237 F F F 5 100-119 137 137 137 237 237 237 F F F 6 120-139 136 136 136 236 236 236 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 130 130 130 227 227 227 T F T <--- 11 220-239 137 137 137 237 237 237 T F T <--- 12 240-259 137 137 137 237 237 237 T F T <--- 13 260-279 134 134 134 233 233 233 F F F 14 280-299 137 137 137 237 237 237 F F F 15 300-319 136 136 136 236 236 236 F F F 16 320-339 138 138 138 238 238 238 T F T <--- 17 340-359 133 133 133 232 232 232 F F F 18 360-379 137 137 137 237 237 237 T F T <--- 19 380-399 134 134 134 234 234 234 F F F 20 400-419 137 137 137 237 237 237 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 138 135 138 240 235 240 F F F <--- 24 480-499 137 137 137 237 237 237 F F F 25 500-519 138 137 138 239 237 239 F F F <--- 26 520-539 130 130 130 227 227 227 T F T <--- 27 540-559 138 137 138 239 237 239 F T F <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 132 138 239 230 239 F F F <--- Max gen: 138, Max flush: 240 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 241 241 241 F F F 1 020-039 137 137 137 237 237 237 F F F 2 040-059 137 137 137 237 237 237 F F F 3 060-079 137 137 137 237 237 237 F F F 4 080-099 137 137 137 237 237 237 F F F 5 100-119 137 137 137 237 237 237 F F F 6 120-139 136 136 136 236 236 236 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 139 139 139 241 241 241 F F F 11 220-239 139 139 139 241 241 241 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 137 137 137 237 237 237 F F F 15 300-319 136 136 136 236 236 236 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 133 133 133 232 232 232 F F F 18 360-379 139 139 139 241 241 241 F F F 19 380-399 134 134 134 234 234 234 F F F 20 400-419 137 137 137 237 237 237 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 138 138 138 240 240 240 F F F 24 480-499 137 137 137 237 237 237 F F F 25 500-519 138 138 138 239 239 239 F F F 26 520-539 139 139 139 241 241 241 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 139, Max flush: 241 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.982943316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.994675354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995004392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995039549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995051758Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995069307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"Crucible 159f4e39-4d86-4b9a-8a73-3bdb410327af has session id: edea6575-eae8-4f15-831f-8c5bf5be6eb8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995428773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Upstairs opts: Upstairs UUID: 159f4e39-4d86-4b9a-8a73-3bdb410327af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.99546257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Crucible stats registered with UUID: 159f4e39-4d86-4b9a-8a73-3bdb410327af","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995478519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:36:59.995492038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995673176Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995709043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995724672Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995821296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995840924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"159f4e39-4d86-4b9a-8a73-3bdb410327af active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995864773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995877802Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995891891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.99590527Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995955177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.995991094Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996006713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33679)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996024902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51803)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996136064Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42533)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996158583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996744463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.99678078Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.996797529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997125627Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997168874Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[0]R flush_numbers[0..12]: [241, 237, 237, 237, 237, 237, 236, 233, 237, 237, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997234749Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[0]R generation[0..12]: [139, 137, 137, 137, 137, 137, 136, 134, 137, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997251898Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997268517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[1]R flush_numbers[0..12]: [241, 237, 237, 237, 237, 237, 236, 233, 237, 237, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997284296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[1]R generation[0..12]: [139, 137, 137, 137, 137, 137, 136, 134, 137, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997300065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997316164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[2]R flush_numbers[0..12]: [241, 237, 237, 237, 237, 237, 236, 233, 237, 237, 241, 241]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997334642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[2]R generation[0..12]: [139, 137, 137, 137, 137, 137, 136, 134, 137, 137, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997350341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.99736641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Max found gen is 140","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-16T21:36:59.997385319Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997399448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997412447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997425756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"":"downstairs","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997439875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997452694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"0","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997466783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"1","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997483072Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"client":"2","":"downstairs","session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997497711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.99751124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"159f4e39-4d86-4b9a-8a73-3bdb410327af is now active with session: edea6575-eae8-4f15-831f-8c5bf5be6eb8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997526139Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997538738Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766,"session_id":"edea6575-eae8-4f15-831f-8c5bf5be6eb8"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997553897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} {"msg":"Activated sub_volume 159f4e39-4d86-4b9a-8a73-3bdb410327af","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:36:59.997565567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 12288 data: 16 15 14 02/30 Read block 260 len 20480 03/30 Read block 208 len 20480 04/30 Read block 158 len 4096 05/30 Read block 52 len 12288 06/30 Write block 40 len 36864 data: 14 11 11 10 9 8 9 9 10 07/30 Write block 287 len 16384 data: 10 11 9 9 08/30 Read block 406 len 4096 09/30 Read block 364 len 20480 10/30 Read block 182 len 32768 11/30 Write block 96 len 32768 data: 13 12 11 11 11 11 11 10 12/30 Read block 556 len 12288 13/30 Write block 109 len 36864 data: 5 6 8 9 8 9 8 9 9 14/30 Read block 159 len 36864 15/30 Read block 21 len 36864 16/30 Write block 73 len 8192 data: 16 13 17/30 Write block 348 len 28672 data: 9 10 10 11 11 11 13 18/30 Read block 362 len 32768 19/30 Write block 103 len 8192 data: 11 8 20/30 Write block 477 len 36864 data: 9 13 13 15 14 13 12 9 10 21/30 Read block 380 len 28672 22/30 Write block 373 len 16384 data: 13 13 16 15 23/30 Write block 537 len 4096 data: 10 24/30 Write block 469 len 24576 data: 12 14 13 12 11 9 25/30 Read block 592 len 20480 26/30 Write block 131 len 12288 data: 8 9 10 27/30 Write block 213 len 32768 data: 6 7 9 10 11 12 14 18 28/30 Write block 246 len 12288 data: 14 9 11 29/30 Read block 355 len 12288 30/30 Read block 434 len 20480 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:00.009700987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1766} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 241 241 241 F F F 1 020-039 137 137 137 237 237 237 F F F 2 040-059 137 137 137 237 237 237 T T F <--- 3 060-079 137 137 137 237 237 237 T T F <--- 4 080-099 137 137 137 237 237 237 T T F <--- 5 100-119 137 137 137 237 237 237 T T F <--- 6 120-139 136 136 136 236 236 236 T T F <--- 7 140-159 134 134 134 233 233 233 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 139 139 139 241 241 241 T T F <--- 11 220-239 139 139 139 241 241 241 T T T 12 240-259 139 139 139 241 241 241 T T F <--- 13 260-279 134 134 134 233 233 233 F F F 14 280-299 137 137 137 237 237 237 T T F <--- 15 300-319 136 136 136 236 236 236 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 133 133 133 232 232 232 T T F <--- 18 360-379 139 139 139 241 241 241 T T F <--- 19 380-399 134 134 134 234 234 234 F F F 20 400-419 137 137 137 237 237 237 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 138 138 138 240 240 240 T T F <--- 24 480-499 137 137 137 237 237 237 T T F <--- 25 500-519 138 138 138 239 239 239 F F F 26 520-539 139 139 139 241 241 241 T T F <--- 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 139, Max flush: 241 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 241 241 241 F F F 1 020-039 137 137 137 237 237 237 F F F 2 040-059 140 140 140 242 242 242 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 140 140 140 242 242 242 F F F 5 100-119 140 140 140 242 242 242 F F F 6 120-139 140 140 140 242 242 242 F F F 7 140-159 134 134 134 233 233 233 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 140 140 242 242 242 F F F 11 220-239 140 140 140 242 242 242 F F F 12 240-259 140 140 140 242 242 242 F F F 13 260-279 134 134 134 233 233 233 F F F 14 280-299 140 140 140 242 242 242 F F F 15 300-319 136 136 136 236 236 236 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 140 140 140 242 242 242 F F F 18 360-379 140 140 140 242 242 242 F F F 19 380-399 134 134 134 234 234 234 F F F 20 400-419 137 137 137 237 237 237 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 140 140 140 242 242 242 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 138 138 138 239 239 239 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 140, Max flush: 242 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.340063434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.351716148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352038656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352074623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352087192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352105211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"Crucible b0ab7a2e-2d10-421b-a2e1-938d4d421b71 has session id: 5ae0ce40-795e-4188-be81-4875dc08f94a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352453307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Upstairs opts: Upstairs UUID: b0ab7a2e-2d10-421b-a2e1-938d4d421b71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352482605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Crucible stats registered with UUID: b0ab7a2e-2d10-421b-a2e1-938d4d421b71","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352497584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:04.352510683Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352693731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352729668Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352747797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352841441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.35285871Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"b0ab7a2e-2d10-421b-a2e1-938d4d421b71 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352875328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352888158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352902077Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352916386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352930045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352944344Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.352958663Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34520)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.353011639Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49377)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.353039287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33739)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.35313939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.353775827Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.353809875Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.353827183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354157751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354204868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[0]R flush_numbers[0..12]: [241, 237, 242, 242, 242, 242, 242, 233, 237, 237, 242, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354277913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[0]R generation[0..12]: [139, 137, 140, 140, 140, 140, 140, 134, 137, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354295041Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.35431165Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[1]R flush_numbers[0..12]: [241, 237, 242, 242, 242, 242, 242, 233, 237, 237, 242, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354327599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[1]R generation[0..12]: [139, 137, 140, 140, 140, 140, 140, 134, 137, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354343068Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354359267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[2]R flush_numbers[0..12]: [241, 237, 242, 242, 242, 242, 242, 233, 237, 237, 242, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354375396Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[2]R generation[0..12]: [139, 137, 140, 140, 140, 140, 140, 134, 137, 137, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354390645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354406794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354420303Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354440212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354453801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.35446897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"":"downstairs","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354483029Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354495818Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"0","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354510487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"1","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354524756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"client":"2","":"downstairs","session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"All required reconciliation work is completed","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-03-16T21:37:04.354541625Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354555314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"b0ab7a2e-2d10-421b-a2e1-938d4d421b71 is now active with session: 5ae0ce40-795e-4188-be81-4875dc08f94a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354570213Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354583092Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774,"session_id":"5ae0ce40-795e-4188-be81-4875dc08f94a"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354596521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} {"msg":"Activated sub_volume b0ab7a2e-2d10-421b-a2e1-938d4d421b71","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:04.354608Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 16384 data: 15 14 13 10 02/30 Read block 486 len 8192 03/30 Write block 510 len 20480 data: 11 10 9 7 9 04/30 Write block 390 len 8192 data: 15 13 05/30 Write block 35 len 40960 data: 13 13 12 16 16 15 12 12 11 10 06/30 Read block 581 len 28672 07/30 Write block 269 len 12288 data: 13 14 13 08/30 Read block 10 len 36864 09/30 Read block 396 len 4096 10/30 Read block 197 len 32768 11/30 Write block 400 len 28672 data: 14 11 14 12 12 12 13 12/30 Read block 64 len 40960 13/30 Read block 415 len 28672 14/30 Read block 408 len 16384 15/30 Read block 148 len 28672 16/30 Write block 239 len 12288 data: 10 9 12 17/30 Read block 441 len 40960 18/30 Write block 362 len 40960 data: 15 12 11 8 13 14 11 11 14 15 19/30 Read block 74 len 4096 20/30 Write block 99 len 40960 data: 12 12 12 12 12 9 9 9 6 6 21/30 Read block 401 len 40960 22/30 Read block 432 len 12288 23/30 Read block 406 len 40960 24/30 Read block 285 len 12288 25/30 Read block 307 len 8192 26/30 Flush 27/30 Write block 291 len 20480 data: 9 8 8 8 9 28/30 Read block 319 len 16384 29/30 Read block 500 len 40960 30/30 Write block 148 len 16384 data: 7 10 9 10 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:04.368186782Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1774} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 241 241 241 F F F 1 020-039 142 137 142 243 237 243 F F F <--- 2 040-059 142 140 142 243 242 243 F F F <--- 3 060-079 140 140 140 242 242 242 F F F 4 080-099 142 140 142 243 242 243 F F F <--- 5 100-119 142 140 142 243 242 243 F F F <--- 6 120-139 140 140 140 242 242 242 F F F 7 140-159 142 134 142 243 233 243 T T T <--- 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 140 140 242 242 242 F F F 11 220-239 142 140 142 243 242 243 F F F <--- 12 240-259 142 140 142 243 242 243 F F F <--- 13 260-279 142 134 142 243 233 243 F F F <--- 14 280-299 140 140 140 242 242 242 T F T <--- 15 300-319 136 136 136 236 236 236 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 140 140 140 242 242 242 F F F 18 360-379 142 140 142 243 242 243 F F F <--- 19 380-399 142 134 142 243 234 243 F F F <--- 20 400-419 142 137 142 243 237 243 F F F <--- 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 140 140 140 242 242 242 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 138 142 243 239 243 F F F <--- 26 520-539 140 140 140 242 242 242 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 142, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 241 241 241 F F F 1 020-039 142 142 142 243 243 243 F F F 2 040-059 142 142 142 243 243 243 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 142 142 142 243 243 243 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 140 140 140 242 242 242 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 137 137 137 237 237 237 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 140 140 242 242 242 F F F 11 220-239 142 142 142 243 243 243 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 142 142 142 243 243 243 F F F 14 280-299 143 143 143 244 244 244 F F F 15 300-319 136 136 136 236 236 236 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 140 140 140 242 242 242 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 142 142 142 243 243 243 F F F 20 400-419 142 142 142 243 243 243 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F F F 23 460-479 140 140 140 242 242 242 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 143, Max flush: 244 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.688767517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:08.699500274Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.69955057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710026694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710358022Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710398559Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710413638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710431847Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"Crucible f4e34b61-847c-43d2-8e37-9af608d5f5bf has session id: 3674de82-d888-4a8a-868f-5631e874324a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710775973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Upstairs opts: Upstairs UUID: f4e34b61-847c-43d2-8e37-9af608d5f5bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710840969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Crucible stats registered with UUID: f4e34b61-847c-43d2-8e37-9af608d5f5bf","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.710857958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:08.710871517Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711039595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711077822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711093571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711194665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711212353Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"f4e34b61-847c-43d2-8e37-9af608d5f5bf active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711232262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711245281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.71125932Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711276249Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711290658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711305177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711319496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51311)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711355654Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40835)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711371852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64402)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.711387031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712229054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712266951Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.71228408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712608468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712649435Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[0]R flush_numbers[0..12]: [241, 243, 243, 242, 243, 243, 242, 244, 237, 237, 242, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712712971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[0]R generation[0..12]: [139, 142, 142, 140, 142, 142, 140, 143, 137, 137, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.71273017Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712746598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[1]R flush_numbers[0..12]: [241, 243, 243, 242, 243, 243, 242, 244, 237, 237, 242, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712764977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[1]R generation[0..12]: [139, 142, 142, 140, 142, 142, 140, 143, 137, 137, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712780746Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712797205Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[2]R flush_numbers[0..12]: [241, 243, 243, 242, 243, 243, 242, 244, 237, 237, 242, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712812864Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[2]R generation[0..12]: [139, 142, 142, 140, 142, 142, 140, 143, 137, 137, 140, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712828383Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712844852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.7128643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712879229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712892588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712906088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"":"downstairs","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712919937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712932276Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"0","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712957334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"1","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712971283Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"client":"2","":"downstairs","session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.712987482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.713001111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"f4e34b61-847c-43d2-8e37-9af608d5f5bf is now active with session: 3674de82-d888-4a8a-868f-5631e874324a","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.71301628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.713028859Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782,"session_id":"3674de82-d888-4a8a-868f-5631e874324a"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.713042158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} {"msg":"Activated sub_volume f4e34b61-847c-43d2-8e37-9af608d5f5bf","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:08.713053727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 380 len 12288 data: 15 11 11 03/30 Read block 346 len 36864 04/30 Flush 05/30 Write block 296 len 40960 data: 11 13 11 8 10 10 9 10 10 8 06/30 Read block 548 len 32768 07/30 Read block 284 len 20480 08/30 Write block 13 len 36864 data: 14 14 13 11 14 13 13 11 11 09/30 Flush 10/30 Write block 160 len 20480 data: 8 11 11 11 11 11/30 Read block 439 len 8192 12/30 Flush 13/30 Flush 14/30 Write block 471 len 12288 data: 14 13 12 15/30 Read block 383 len 8192 16/30 Read block 47 len 32768 17/30 Read block 52 len 24576 18/30 Read block 53 len 20480 19/30 Read block 405 len 12288 20/30 Flush 21/30 Write block 396 len 8192 data: 16 16 22/30 Read block 431 len 20480 23/30 Read block 467 len 16384 24/30 Read block 510 len 12288 25/30 Write block 527 len 36864 data: 8 10 10 10 9 9 11 11 8 26/30 Read block 223 len 36864 27/30 Read block 28 len 28672 28/30 Read block 549 len 8192 29/30 Write block 225 len 40960 data: 17 16 19 17 16 19 17 17 16 17 30/30 Write block 453 len 36864 data: 15 15 12 16 16 14 15 15 10 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 1 27 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:08.726123634Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1782} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 144 144 241 247 247 F F F <--- 1 020-039 142 144 144 243 247 247 F F F <--- 2 040-059 142 142 142 243 243 243 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 142 142 142 243 243 243 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 140 140 140 242 242 242 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 137 144 144 237 248 248 F F F <--- 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 140 140 242 242 242 F F F 11 220-239 142 142 142 243 243 243 F T T <--- 12 240-259 142 142 142 243 243 243 F F F 13 260-279 142 142 142 243 243 243 F F F 14 280-299 143 144 144 244 247 247 F F F <--- 15 300-319 136 144 144 236 247 247 F F F <--- 16 320-339 139 139 139 241 241 241 F F F 17 340-359 140 140 140 242 242 242 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 142 144 144 243 246 246 T T T <--- 20 400-419 142 142 142 243 243 243 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 122 122 122 212 212 212 F T T <--- 23 460-479 140 144 144 242 250 250 F T T <--- 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 140 140 140 242 242 242 F T T <--- 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 144, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 144 144 144 247 247 247 F F F 2 040-059 142 142 142 243 243 243 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 142 142 142 243 243 243 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 140 140 140 242 242 242 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 140 140 242 242 242 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 142 142 142 243 243 243 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 144 144 144 247 247 247 F F F 16 320-339 139 139 139 241 241 241 F F F 17 340-359 140 140 140 242 242 242 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 145 145 145 251 251 251 F F F 20 400-419 142 142 142 243 243 243 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 145 145 145 251 251 251 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 145 145 145 251 251 251 F F F 27 540-559 138 138 138 239 239 239 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 138 138 239 239 239 F F F Max gen: 145, Max flush: 251 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.058607756Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:13.069317374Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.06937492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.079603801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.079903271Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.079943388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.079955477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.079973156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"Crucible d9ceae6a-915a-4ae1-9603-4d569f37f6b1 has session id: 7801ae26-9374-489e-9099-4189d4f4f603","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080309763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Upstairs opts: Upstairs UUID: d9ceae6a-915a-4ae1-9603-4d569f37f6b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08034507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Crucible stats registered with UUID: d9ceae6a-915a-4ae1-9603-4d569f37f6b1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080363759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:13.080377448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080534807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080559426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080574175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080684797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080702766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"d9ceae6a-915a-4ae1-9603-4d569f37f6b1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080724844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080737544Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080751473Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080765482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080779221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08079375Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080808259Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"ds_connection connected from Ok(127.0.0.1:39507)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080841896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"ds_connection connected from Ok(127.0.0.1:36599)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.080859595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"ds_connection connected from Ok(127.0.0.1:41178)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.081000286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.081635892Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08166482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.081723126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082003477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082041444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 243, 242, 243, 243, 242, 244, 248, 237, 242, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08210504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[0]R generation[0..12]: [144, 144, 142, 140, 142, 142, 140, 143, 144, 137, 140, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082122509Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082139478Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 243, 242, 243, 243, 242, 244, 248, 237, 242, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082155547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[1]R generation[0..12]: [144, 144, 142, 140, 142, 142, 140, 143, 144, 137, 140, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082171526Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082187584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 243, 242, 243, 243, 242, 244, 248, 237, 242, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082203103Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[2]R generation[0..12]: [144, 144, 142, 140, 142, 142, 140, 143, 144, 137, 140, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082218662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082237551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08225766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082271659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082285038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082298557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"":"downstairs","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082312956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082325515Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"0","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082339964Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"1","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:37:13.082354013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"client":"2","":"downstairs","session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082368442Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082381791Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"d9ceae6a-915a-4ae1-9603-4d569f37f6b1 is now active with session: 7801ae26-9374-489e-9099-4189d4f4f603","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.08239643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082409209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790,"session_id":"7801ae26-9374-489e-9099-4189d4f4f603"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082422748Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} {"msg":"Activated sub_volume d9ceae6a-915a-4ae1-9603-4d569f37f6b1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:13.082434198Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 137 len 20480 data: 13 15 12 16 15 03/30 Read block 267 len 40960 04/30 Write block 580 len 4096 data: 7 05/30 Write block 313 len 4096 data: 14 06/30 Write block 79 len 12288 data: 8 7 10 07/30 Read block 247 len 12288 08/30 Read block 270 len 16384 09/30 Write block 357 len 24576 data: 18 16 17 15 18 16 10/30 Flush 11/30 Write block 207 len 36864 data: 17 16 16 14 7 6 7 8 10 12/30 Flush 13/30 Write block 134 len 32768 data: 11 10 14 14 16 13 17 16 14/30 Write block 326 len 32768 data: 8 9 8 10 9 11 11 10 15/30 Read block 413 len 28672 16/30 Write block 408 len 36864 data: 16 16 13 14 14 12 12 10 10 17/30 Read block 315 len 20480 18/30 Read block 321 len 8192 19/30 Write block 78 len 20480 data: 7 9 8 11 8 20/30 Read block 402 len 40960 21/30 Read block 280 len 20480 22/30 Write block 551 len 20480 data: 6 6 9 10 11 23/30 Write block 338 len 8192 data: 11 9 24/30 Read block 526 len 12288 25/30 Read block 560 len 12288 26/30 Read block 438 len 36864 27/30 Write block 330 len 28672 data: 10 12 12 11 10 15 15 28/30 Write block 293 len 4096 data: 9 29/30 Write block 398 len 16384 data: 17 14 15 12 30/30 Write block 279 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 7 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 4 4 37 Done 0 25 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:13.09454133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1790} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 144 144 144 247 247 247 F F F 2 040-059 142 142 142 243 243 243 F F F 3 060-079 140 146 146 242 253 253 F T T <--- 4 080-099 142 146 146 243 253 253 F T T <--- 5 100-119 142 142 142 243 243 243 F F F 6 120-139 140 146 146 242 253 253 F T T <--- 7 140-159 143 146 146 244 253 253 F T T <--- 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 140 146 146 242 254 254 F F F <--- 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 142 142 142 243 243 243 F T T <--- 14 280-299 144 144 144 247 247 247 F T T <--- 15 300-319 144 146 146 247 253 253 F F F <--- 16 320-339 139 139 139 241 241 241 F T T <--- 17 340-359 140 146 146 242 253 253 F F F <--- 18 360-379 142 146 146 243 253 253 F F F <--- 19 380-399 145 145 145 251 251 251 F T T <--- 20 400-419 142 142 142 243 243 243 F T T <--- 21 420-439 134 134 134 233 233 233 F F F 22 440-459 145 145 145 251 251 251 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 145 145 145 251 251 251 F F F 27 540-559 138 138 138 239 239 239 F T T <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 138 146 146 239 253 253 F F F <--- Max gen: 146, Max flush: 254 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 144 144 144 247 247 247 F F F 2 040-059 142 142 142 243 243 243 F F F 3 060-079 147 147 147 255 255 255 F F F 4 080-099 147 147 147 255 255 255 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 146 146 146 254 254 254 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 146 146 253 253 253 F F F 16 320-339 147 147 147 255 255 255 F F F 17 340-359 146 146 146 253 253 253 F F F 18 360-379 146 146 146 253 253 253 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 147 147 147 255 255 255 F F F 21 420-439 134 134 134 233 233 233 F F F 22 440-459 145 145 145 251 251 251 F F F 23 460-479 145 145 145 251 251 251 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 145 145 145 251 251 251 F F F 27 540-559 147 147 147 255 255 255 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 147, Max flush: 255 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.434989457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:17.445698766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.445751872Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456012621Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456310401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456349848Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456362257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456379956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"Crucible 142990e2-28a8-4129-8b2e-d06392c4083b has session id: 39d4c70e-fcf6-448f-90e4-b317f1a89816","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.45675525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Upstairs opts: Upstairs UUID: 142990e2-28a8-4129-8b2e-d06392c4083b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456798257Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Crucible stats registered with UUID: 142990e2-28a8-4129-8b2e-d06392c4083b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456813976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:17.456827495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.456989844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457015922Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457030681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457150363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457168472Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"142990e2-28a8-4129-8b2e-d06392c4083b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457185381Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.45719819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457212489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457226818Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457240507Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457281574Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457304513Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"ds_connection connected from Ok(127.0.0.1:62009)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457320262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"ds_connection connected from Ok(127.0.0.1:47760)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457335421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"ds_connection connected from Ok(127.0.0.1:43903)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.457395286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458088089Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458125567Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458143075Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458463603Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.45850799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 243, 255, 255, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458576076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[0]R generation[0..12]: [144, 144, 142, 147, 147, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458593285Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458609923Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 243, 255, 255, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458625622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[1]R generation[0..12]: [144, 144, 142, 147, 147, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458641371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.45865762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 243, 255, 255, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458673199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[2]R generation[0..12]: [144, 144, 142, 147, 147, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458688858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458705107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458718596Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458738495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458751724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458765033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"":"downstairs","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458780832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458793501Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"0","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.45880739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"1","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458823729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"client":"2","":"downstairs","session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458838988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458852537Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"142990e2-28a8-4129-8b2e-d06392c4083b is now active with session: 39d4c70e-fcf6-448f-90e4-b317f1a89816","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458867506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458880555Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798,"session_id":"39d4c70e-fcf6-448f-90e4-b317f1a89816"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458893804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} {"msg":"Activated sub_volume 142990e2-28a8-4129-8b2e-d06392c4083b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:17.458905183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 32768 data: 10 9 8 10 9 12 9 6 02/30 Read block 139 len 36864 03/30 Write block 441 len 16384 data: 10 10 12 9 04/30 Read block 411 len 8192 05/30 Write block 464 len 28672 data: 9 9 11 12 12 13 15 06/30 Write block 526 len 24576 data: 6 9 11 11 11 10 07/30 Read block 108 len 36864 08/30 Read block 374 len 8192 09/30 Write block 446 len 28672 data: 12 10 11 7 10 13 18 10/30 Read block 199 len 40960 11/30 Read block 29 len 4096 12/30 Read block 73 len 16384 13/30 Read block 114 len 40960 14/30 Flush 15/30 Write block 368 len 8192 data: 12 12 16/30 Write block 476 len 20480 data: 7 10 14 14 16 17/30 Write block 24 len 16384 data: 12 10 10 11 18/30 Write block 430 len 28672 data: 15 14 12 10 11 11 10 19/30 Read block 385 len 8192 20/30 Read block 392 len 16384 21/30 Write block 97 len 4096 data: 13 22/30 Flush 23/30 Write block 457 len 24576 data: 17 15 16 16 11 10 24/30 Read block 567 len 24576 25/30 Write block 541 len 16384 data: 11 12 10 9 26/30 Read block 562 len 36864 27/30 Read block 23 len 40960 28/30 Read block 375 len 40960 29/30 Read block 297 len 20480 30/30 Read block 517 len 28672 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 0 23 Done 8 29 30 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:17.471339124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1798} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 144 148 148 247 257 257 F F F <--- 2 040-059 142 142 142 243 243 243 F F F 3 060-079 147 148 148 255 256 256 T F F <--- 4 080-099 147 148 148 255 257 257 T F F <--- 5 100-119 142 142 142 243 243 243 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 146 146 146 254 254 254 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 146 146 253 253 253 F F F 16 320-339 147 147 147 255 255 255 F F F 17 340-359 146 146 146 253 253 253 F F F 18 360-379 146 148 148 253 257 257 F F F <--- 19 380-399 147 147 147 255 255 255 F F F 20 400-419 147 147 147 255 255 255 F F F 21 420-439 134 148 148 233 257 257 F F F <--- 22 440-459 145 148 148 251 256 256 T T T <--- 23 460-479 145 148 148 251 257 257 T T T <--- 24 480-499 140 148 148 242 257 257 F F F <--- 25 500-519 142 142 142 243 243 243 F F F 26 520-539 145 148 148 251 256 256 T F F <--- 27 540-559 147 147 147 255 255 255 F T T <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 148, Max flush: 257 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 148 148 148 257 257 257 F F F 2 040-059 142 142 142 243 243 243 F F F 3 060-079 148 148 148 256 256 256 F F F 4 080-099 148 148 148 257 257 257 F F F 5 100-119 142 142 142 243 243 243 F F F 6 120-139 147 147 147 255 255 255 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 146 146 146 254 254 254 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 146 146 253 253 253 F F F 16 320-339 147 147 147 255 255 255 F F F 17 340-359 146 146 146 253 253 253 F F F 18 360-379 148 148 148 257 257 257 F F F 19 380-399 147 147 147 255 255 255 F F F 20 400-419 147 147 147 255 255 255 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 149 149 149 258 258 258 F F F 23 460-479 149 149 149 258 258 258 F F F 24 480-499 148 148 148 257 257 257 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 149 149 149 258 258 258 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 149, Max flush: 258 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.79774174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:21.808446139Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.808504485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.818858238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819152967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819191605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819206804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819224903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"Crucible 9e14bdde-5004-48ac-b5ac-d7e8be8662e1 has session id: 1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.81956108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Upstairs opts: Upstairs UUID: 9e14bdde-5004-48ac-b5ac-d7e8be8662e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819595637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Crucible stats registered with UUID: 9e14bdde-5004-48ac-b5ac-d7e8be8662e1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819611486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:21.819625015Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819800213Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819835781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.8198514Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819946313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.819964232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"9e14bdde-5004-48ac-b5ac-d7e8be8662e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.81999092Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820004599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820018468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820032227Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820048866Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820063685Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"ds_connection connected from Ok(127.0.0.1:39797)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820078634Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"ds_connection connected from Ok(127.0.0.1:53686)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820093703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820107892Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"ds_connection connected from Ok(127.0.0.1:60537)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820235983Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.820816914Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.82087899Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821002891Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821179549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821219816Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[0]R flush_numbers[0..12]: [247, 257, 243, 256, 257, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821380175Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[0]R generation[0..12]: [144, 148, 142, 148, 148, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821397804Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821414553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[1]R flush_numbers[0..12]: [247, 257, 243, 256, 257, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821430152Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[1]R generation[0..12]: [144, 148, 142, 148, 148, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821448531Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821465199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[2]R flush_numbers[0..12]: [247, 257, 243, 256, 257, 243, 255, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821480868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[2]R generation[0..12]: [144, 148, 142, 148, 148, 142, 147, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821496807Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821513256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821532265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821546504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821559643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821572882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"":"downstairs","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821587111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.82160039Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"0","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821614629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"1","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2026-03-16T21:37:21.821629028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"client":"2","":"downstairs","session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821646637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821662426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"9e14bdde-5004-48ac-b5ac-d7e8be8662e1 is now active with session: 1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821677705Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821691554Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806,"session_id":"1d9df92b-fbb5-4a1a-b112-09ec5eb7bd52"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821705023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} {"msg":"Activated sub_volume 9e14bdde-5004-48ac-b5ac-d7e8be8662e1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:21.821716272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 492 len 20480 data: 11 12 14 12 9 02/30 Write block 394 len 28672 data: 14 15 17 17 18 15 16 03/30 Read block 255 len 12288 04/30 Read block 174 len 4096 05/30 Read block 258 len 32768 06/30 Read block 447 len 20480 07/30 Read block 12 len 40960 08/30 Write block 128 len 20480 data: 10 10 9 9 10 09/30 Read block 118 len 36864 10/30 Write block 91 len 20480 data: 7 6 8 7 9 11/30 Read block 234 len 4096 12/30 Write block 487 len 4096 data: 16 13/30 Read block 338 len 8192 14/30 Flush 15/30 Write block 456 len 32768 data: 17 18 16 17 17 12 11 10 16/30 Write block 44 len 12288 data: 11 9 10 17/30 Read block 477 len 24576 18/30 Read block 471 len 24576 19/30 Read block 272 len 4096 20/30 Read block 25 len 8192 21/30 Write block 101 len 16384 data: 13 13 13 10 22/30 Write block 63 len 8192 data: 17 19 23/30 Read block 457 len 40960 24/30 Read block 180 len 36864 25/30 Read block 590 len 4096 26/30 Read block 512 len 32768 27/30 Write block 376 len 24576 data: 16 14 15 15 16 12 28/30 Read block 109 len 36864 29/30 Read block 529 len 8192 30/30 Write block 338 len 20480 data: 12 10 8 10 8 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:21.833514266Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1806} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 148 148 148 257 257 257 F F F 2 040-059 142 142 142 243 243 243 F T T <--- 3 060-079 148 148 148 256 256 256 F T T <--- 4 080-099 148 150 150 257 259 259 F F F <--- 5 100-119 142 142 142 243 243 243 F T T <--- 6 120-139 147 150 150 255 259 259 F F F <--- 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 146 146 146 254 254 254 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 146 146 253 253 253 F F F 16 320-339 147 147 147 255 255 255 F T T <--- 17 340-359 146 146 146 253 253 253 F T T <--- 18 360-379 148 148 148 257 257 257 F T T <--- 19 380-399 147 150 150 255 259 259 T T T <--- 20 400-419 147 150 150 255 259 259 T F F <--- 21 420-439 148 148 148 257 257 257 F F F 22 440-459 149 149 149 258 258 258 F T T <--- 23 460-479 149 149 149 258 258 258 F T T <--- 24 480-499 148 150 150 257 259 259 T F F <--- 25 500-519 142 142 142 243 243 243 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 149 149 149 258 258 258 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 150, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 148 148 148 257 257 257 F F F 2 040-059 151 151 151 260 260 260 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 150 150 150 259 259 259 F F F 5 100-119 151 151 151 260 260 260 F F F 6 120-139 150 150 150 259 259 259 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 144 144 248 248 248 F F F 9 180-199 137 137 137 237 237 237 F F F 10 200-219 146 146 146 254 254 254 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 146 146 253 253 253 F F F 16 320-339 151 151 151 260 260 260 F F F 17 340-359 151 151 151 260 260 260 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 F F F 20 400-419 150 150 150 259 259 259 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 F F F 23 460-479 151 151 151 260 260 260 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 148 148 148 256 256 256 F F F 27 540-559 149 149 149 258 258 258 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 151, Max flush: 260 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.169705424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:26.180398663Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.18045286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.190705079Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191003489Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191043486Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191055625Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191073564Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"Crucible 9df61e2f-4bac-4a22-9048-f58d7a361446 has session id: d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191389862Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Upstairs opts: Upstairs UUID: 9df61e2f-4bac-4a22-9048-f58d7a361446, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191417391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Crucible stats registered with UUID: 9df61e2f-4bac-4a22-9048-f58d7a361446","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191434929Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:26.191448288Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191630706Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191655384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191670103Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191812214Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191831422Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"9df61e2f-4bac-4a22-9048-f58d7a361446 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191854581Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.19186749Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191881669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191895528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"2","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191909107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191923316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191937485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51270)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.191976742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47025)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.192118713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43174)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.19215131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.192649426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.192756239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"2","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.192901749Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193022171Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193086247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[0]R flush_numbers[0..12]: [247, 257, 260, 260, 259, 260, 259, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193292192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[0]R generation[0..12]: [144, 148, 151, 151, 150, 151, 150, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193309661Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.19332623Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[1]R flush_numbers[0..12]: [247, 257, 260, 260, 259, 260, 259, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193342519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[1]R generation[0..12]: [144, 148, 151, 151, 150, 151, 150, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193358278Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193374627Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[2]R flush_numbers[0..12]: [247, 257, 260, 260, 259, 260, 259, 255, 248, 237, 254, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193390286Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[2]R generation[0..12]: [144, 148, 151, 151, 150, 151, 150, 147, 144, 137, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193405715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193425263Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Max found gen is 152","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-03-16T21:37:26.193444412Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193458861Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.19347247Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193485839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"":"downstairs","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193499728Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193512307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"0","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193526316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"1","":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193540326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"client":"2","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"downstairs","session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193557304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193571033Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"9df61e2f-4bac-4a22-9048-f58d7a361446 is now active with session: d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193585932Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193598542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814,"session_id":"d7e2b472-053c-4c6e-b0e8-ae8a2445e2a1"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.193611801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} {"msg":"Activated sub_volume 9df61e2f-4bac-4a22-9048-f58d7a361446","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:26.19362315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 197 len 28672 data: 11 12 12 13 14 16 16 03/30 Read block 538 len 28672 04/30 Read block 158 len 16384 05/30 Read block 549 len 40960 06/30 Write block 49 len 36864 data: 10 9 9 10 8 7 7 10 10 07/30 Read block 204 len 16384 08/30 Read block 417 len 36864 09/30 Read block 526 len 16384 10/30 Read block 463 len 40960 11/30 Write block 178 len 40960 data: 11 14 15 16 15 13 12 11 10 7 12/30 Write block 49 len 16384 data: 11 10 10 11 13/30 Write block 37 len 16384 data: 13 17 17 16 14/30 Write block 343 len 20480 data: 7 7 7 10 10 15/30 Write block 537 len 36864 data: 11 9 8 9 12 13 11 10 8 16/30 Read block 67 len 36864 17/30 Read block 52 len 40960 18/30 Write block 315 len 12288 data: 15 15 16 19/30 Read block 582 len 28672 20/30 Read block 38 len 24576 21/30 Write block 536 len 28672 data: 8 12 10 9 10 13 14 22/30 Read block 26 len 4096 23/30 Write block 132 len 32768 data: 11 11 12 11 15 15 17 14 24/30 Read block 95 len 16384 25/30 Flush 26/30 Write block 263 len 8192 data: 11 10 27/30 Read block 516 len 24576 28/30 Read block 552 len 24576 29/30 Write block 338 len 32768 data: 13 11 9 11 9 8 8 8 30/30 Write block 548 len 20480 data: 9 9 7 7 7 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:26.206638901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1814} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 148 152 152 257 262 262 F F F <--- 2 040-059 151 152 152 260 262 262 F F F <--- 3 060-079 151 151 151 260 260 260 F F F 4 080-099 150 150 150 259 259 259 F F F 5 100-119 151 151 151 260 260 260 F F F 6 120-139 150 152 152 259 262 262 F F F <--- 7 140-159 147 147 147 255 255 255 F F F 8 160-179 144 152 152 248 262 262 F F F <--- 9 180-199 137 152 152 237 262 262 F F F <--- 10 200-219 146 152 152 254 262 262 F F F <--- 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 147 147 147 255 255 255 F T T <--- 14 280-299 147 147 147 255 255 255 F F F 15 300-319 146 152 152 253 262 262 F F F <--- 16 320-339 151 151 151 260 260 260 F T T <--- 17 340-359 151 152 152 260 262 262 F T T <--- 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 F F F 20 400-419 150 150 150 259 259 259 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 F F F 23 460-479 151 151 151 260 260 260 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 148 152 152 256 262 262 F F F <--- 27 540-559 149 152 152 258 262 262 F T T <--- 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 152, Max flush: 262 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 152 152 152 262 262 262 F F F 2 040-059 152 152 152 262 262 262 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 150 150 150 259 259 259 F F F 5 100-119 151 151 151 260 260 260 F F F 6 120-139 152 152 152 262 262 262 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 152 152 152 262 262 262 F F F 10 200-219 152 152 152 262 262 262 F F F 11 220-239 145 145 145 251 251 251 F F F 12 240-259 142 142 142 243 243 243 F F F 13 260-279 153 153 153 263 263 263 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 153 153 153 263 263 263 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 F F F 20 400-419 150 150 150 259 259 259 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 F F F 23 460-479 151 151 151 260 260 260 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 142 142 142 243 243 243 F F F 26 520-539 152 152 152 262 262 262 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 129 129 129 225 225 225 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 153, Max flush: 263 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.556050316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.56798242Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568303798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568341616Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568353855Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568371444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"Crucible b9a51a51-9e48-432f-aabf-2fa57b97d0ca has session id: bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568706881Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Upstairs opts: Upstairs UUID: b9a51a51-9e48-432f-aabf-2fa57b97d0ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568734529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Crucible stats registered with UUID: b9a51a51-9e48-432f-aabf-2fa57b97d0ca","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568749278Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:30.568762617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568945044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.568991461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.56900775Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569101514Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569119223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"b9a51a51-9e48-432f-aabf-2fa57b97d0ca active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569142031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.5691576Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569172569Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569187188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569201107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569215496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569229645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57334)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569278632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64667)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.569294701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33338)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.56930987Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570044659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570072307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570088996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570454781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.57047607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[0]R flush_numbers[0..12]: [247, 262, 262, 260, 259, 260, 262, 255, 262, 262, 262, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570491799Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[0]R generation[0..12]: [144, 152, 152, 151, 150, 151, 152, 147, 152, 152, 152, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570511197Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570527916Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[1]R flush_numbers[0..12]: [247, 262, 262, 260, 259, 260, 262, 255, 262, 262, 262, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570543635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[1]R generation[0..12]: [144, 152, 152, 151, 150, 151, 152, 147, 152, 152, 152, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570559644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570576063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[2]R flush_numbers[0..12]: [247, 262, 262, 260, 259, 260, 262, 255, 262, 262, 262, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570591862Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[2]R generation[0..12]: [144, 152, 152, 151, 150, 151, 152, 147, 152, 152, 152, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570607551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.57062401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570642828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570657028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570670597Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570684146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"":"downstairs","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570698375Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570711194Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"0","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570727083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"clientDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"1","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570744292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"client":"2","":"downstairs","session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570758451Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.57077178Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"b9a51a51-9e48-432f-aabf-2fa57b97d0ca is now active with session: bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570786389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570799028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822,"session_id":"bc3f7a5f-a4fa-4020-87d9-0a0fa8dc844f"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570812307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} {"msg":"Activated sub_volume b9a51a51-9e48-432f-aabf-2fa57b97d0ca","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:30.570823466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 8192 data: 10 9 02/30 Read block 9 len 28672 03/30 Flush 04/30 Write block 91 len 40960 data: 8 7 9 8 10 14 14 12 13 13 05/30 Flush 06/30 Write block 188 len 16384 data: 6 6 7 8 07/30 Read block 154 len 32768 08/30 Flush 09/30 Write block 94 len 28672 data: 9 11 15 15 13 14 14 10/30 Read block 55 len 8192 11/30 Flush 12/30 Write block 261 len 24576 data: 11 10 12 11 9 10 13/30 Read block 18 len 12288 14/30 Read block 481 len 32768 15/30 Read block 246 len 40960 16/30 Read block 313 len 32768 17/30 Read block 459 len 4096 18/30 Write block 575 len 4096 data: 10 19/30 Read block 350 len 8192 20/30 Write block 11 len 32768 data: 14 16 15 15 14 12 15 14 21/30 Write block 224 len 20480 data: 19 18 17 20 18 22/30 Write block 45 len 8192 data: 10 11 23/30 Write block 124 len 40960 data: 13 12 13 12 11 11 10 10 12 12 24/30 Write block 120 len 12288 data: 12 10 10 25/30 Read block 388 len 40960 26/30 Write block 246 len 36864 data: 15 10 12 11 10 9 6 7 7 27/30 Read block 307 len 24576 28/30 Write block 1 len 28672 data: 5 7 9 8 10 12 12 29/30 Write block 502 len 36864 data: 18 23 21 21 19 20 16 13 12 30/30 Write block 515 len 12288 data: 11 12 14 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:30.582931569Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1822} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 T T F <--- 1 020-039 152 152 152 262 262 262 F F F 2 040-059 152 152 152 262 262 262 T T F <--- 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 150 267 267 259 F F F <--- 5 100-119 154 154 151 267 267 260 F F F <--- 6 120-139 152 152 152 262 262 262 T T F <--- 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 152 266 266 262 F F F <--- 10 200-219 152 152 152 262 262 262 F F F 11 220-239 145 145 145 251 251 251 T T F <--- 12 240-259 142 142 142 243 243 243 T T F <--- 13 260-279 153 153 153 263 263 263 T T F <--- 14 280-299 147 147 147 255 255 255 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 153 153 153 263 263 263 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 F F F 20 400-419 150 150 150 259 259 259 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 F F F 23 460-479 151 151 151 260 260 260 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 142 142 142 243 243 243 T T F <--- 26 520-539 152 152 152 262 262 262 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 154 154 129 264 264 225 T T T <--- 29 580-599 146 146 146 253 253 253 F F F Max gen: 154, Max flush: 267 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 152 152 152 262 262 262 F F F 2 040-059 155 155 155 268 268 268 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 154 154 154 267 267 267 F F F 6 120-139 155 155 155 268 268 268 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F F F 10 200-219 152 152 152 262 262 262 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 155 155 155 268 268 268 F F F 14 280-299 147 147 147 255 255 255 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 153 153 153 263 263 263 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 F F F 20 400-419 150 150 150 259 259 259 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 F F F 23 460-479 151 151 151 260 260 260 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 155 155 155 268 268 268 F F F 26 520-539 152 152 152 262 262 262 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 155, Max flush: 268 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.904661994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.916380134Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.916708551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.916753588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.916766057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.916783906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"Crucible d15e3ede-2fd3-4b8f-9385-5847d788cb02 has session id: a244a7a2-e471-49ab-8534-893062b8cd05","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917125553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Upstairs opts: Upstairs UUID: d15e3ede-2fd3-4b8f-9385-5847d788cb02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.9171699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Crucible stats registered with UUID: d15e3ede-2fd3-4b8f-9385-5847d788cb02","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917185909Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:34.917199438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917386305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917422873Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917438811Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917540294Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917558633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"d15e3ede-2fd3-4b8f-9385-5847d788cb02 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917582142Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917595181Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.91760907Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917623059Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917636598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917650947Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917665236Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"ds_connection connected from Ok(127.0.0.1:41875)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917690284Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"ds_connection connected from Ok(127.0.0.1:42285)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917707733Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"ds_connection connected from Ok(127.0.0.1:35867)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.917722822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918458712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.91848769Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918511828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918821607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918867654Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[0]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 262, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918908281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[0]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 152, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.91892535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918941819Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[1]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 262, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918957568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[1]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 152, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918973267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.918989535Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[2]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 262, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919005614Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[2]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 152, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919023953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919040482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919059411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.91907397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919087359Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919100718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"":"downstairs","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919114837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919127506Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"0","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919141985Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"1","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919155664Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"client":"2","":"downstairs","session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919172773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919186132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"d15e3ede-2fd3-4b8f-9385-5847d788cb02 is now active with session: a244a7a2-e471-49ab-8534-893062b8cd05","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919201021Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.91921368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830,"session_id":"a244a7a2-e471-49ab-8534-893062b8cd05"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919227389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} {"msg":"Activated sub_volume d15e3ede-2fd3-4b8f-9385-5847d788cb02","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:34.919240278Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 32768 data: 18 18 19 16 17 13 15 13 02/30 Read block 432 len 4096 03/30 Read block 61 len 24576 04/30 Read block 462 len 28672 05/30 Write block 385 len 8192 data: 14 16 06/30 Write block 415 len 8192 data: 11 11 07/30 Read block 168 len 8192 08/30 Read block 508 len 36864 09/30 Write block 209 len 20480 data: 17 15 8 7 8 10/30 Write block 353 len 16384 data: 12 14 18 19 11/30 Write block 515 len 40960 data: 12 13 15 15 13 13 11 10 5 5 12/30 Write block 396 len 36864 data: 19 19 20 17 18 14 16 14 13 13/30 Read block 331 len 16384 14/30 Read block 5 len 32768 15/30 Read block 134 len 4096 16/30 Write block 274 len 36864 data: 10 10 7 9 9 9 10 12 11 17/30 Read block 564 len 12288 18/30 Write block 456 len 8192 data: 18 19 19/30 Write block 523 len 40960 data: 6 6 5 7 10 12 12 12 11 10 20/30 Read block 256 len 32768 21/30 Read block 583 len 36864 22/30 Read block 453 len 20480 23/30 Read block 178 len 40960 24/30 Write block 510 len 36864 data: 13 11 10 8 10 13 14 16 16 25/30 Read block 284 len 12288 26/30 Read block 316 len 32768 27/30 Write block 466 len 28672 data: 12 13 13 14 16 15 14 28/30 Read block 274 len 40960 29/30 Read block 300 len 8192 30/30 Read block 441 len 4096 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 24 1 25 Done 30 6 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:34.931963419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1830} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 152 152 152 262 262 262 F F F 2 040-059 155 155 155 268 268 268 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 154 154 154 267 267 267 F F F 6 120-139 155 155 155 268 268 268 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F F F 10 200-219 152 152 152 262 262 262 T F T <--- 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 155 155 155 268 268 268 T F T <--- 14 280-299 147 147 147 255 255 255 T F T <--- 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 153 153 153 263 263 263 T F T <--- 18 360-379 151 151 151 260 260 260 F F F 19 380-399 151 151 151 260 260 260 T T T 20 400-419 150 150 150 259 259 259 T T T 21 420-439 148 148 148 257 257 257 F F F 22 440-459 151 151 151 260 260 260 T F T <--- 23 460-479 151 151 151 260 260 260 T F T <--- 24 480-499 150 150 150 259 259 259 F F F 25 500-519 155 155 155 268 268 268 T F T <--- 26 520-539 152 152 152 262 262 262 T F T <--- 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 155, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 152 152 152 262 262 262 F F F 2 040-059 155 155 155 268 268 268 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 154 154 154 267 267 267 F F F 6 120-139 155 155 155 268 268 268 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 156 156 156 269 269 269 F F F 14 280-299 156 156 156 269 269 269 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 156 156 156 269 269 269 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 156 156 156 269 269 269 F F F 23 460-479 156 156 156 269 269 269 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 156 156 156 269 269 269 F F F 26 520-539 156 156 156 269 269 269 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 146 146 146 253 253 253 F F F Max gen: 156, Max flush: 269 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.253609141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:39.264294671Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.264346727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.274798413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275106212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.27514457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275156729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275174598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"Crucible cc7a0b33-1fa9-4243-8233-6a507bce2ee9 has session id: d7746b55-cc43-46f3-9c43-7fd424ab3544","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275482426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Upstairs opts: Upstairs UUID: cc7a0b33-1fa9-4243-8233-6a507bce2ee9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275538933Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Crucible stats registered with UUID: cc7a0b33-1fa9-4243-8233-6a507bce2ee9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275555801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:39.275569641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275738139Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275767127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275782116Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275892168Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275910107Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"cc7a0b33-1fa9-4243-8233-6a507bce2ee9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275926776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275939505Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275953584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275967833Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275981852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.275995921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.27601028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"ds_connection connected from Ok(127.0.0.1:45537)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.276025429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"ds_connection connected from Ok(127.0.0.1:39775)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.276040648Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"ds_connection connected from Ok(127.0.0.1:61440)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.276179849Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.276694564Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.27674963Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.276959976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277059449Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277096186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[0]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277370867Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[0]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277390076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277406525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[1]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277422354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[1]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277437963Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277454432Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[2]R flush_numbers[0..12]: [268, 262, 268, 260, 267, 267, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277470121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[2]R generation[0..12]: [155, 152, 155, 151, 154, 154, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.27748557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277501639Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:37:39.277515308Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277535316Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277551025Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277564984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"":"downstairs","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277579003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277591732Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"0","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277605781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"1","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.2776193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"client":"2","":"downstairs","session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:37:39.277633689Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277649038Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"cc7a0b33-1fa9-4243-8233-6a507bce2ee9 is now active with session: d7746b55-cc43-46f3-9c43-7fd424ab3544","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277663977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277676857Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838,"session_id":"d7746b55-cc43-46f3-9c43-7fd424ab3544"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277690126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} {"msg":"Activated sub_volume cc7a0b33-1fa9-4243-8233-6a507bce2ee9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:39.277701385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 32768 data: 10 9 9 9 11 11 10 11 02/30 Read block 118 len 24576 03/30 Read block 450 len 8192 04/30 Read block 272 len 28672 05/30 Read block 530 len 40960 06/30 Read block 567 len 16384 07/30 Read block 275 len 32768 08/30 Read block 504 len 16384 09/30 Read block 554 len 28672 10/30 Read block 446 len 12288 11/30 Write block 26 len 8192 data: 11 12 12/30 Read block 443 len 24576 13/30 Read block 101 len 12288 14/30 Flush 15/30 Write block 445 len 28672 data: 11 13 11 12 8 11 14 16/30 Read block 143 len 4096 17/30 Write block 45 len 32768 data: 11 12 10 11 12 11 11 12 18/30 Write block 110 len 4096 data: 7 19/30 Read block 316 len 4096 20/30 Write block 37 len 20480 data: 14 18 18 17 13 21/30 Write block 55 len 20480 data: 8 11 11 10 10 22/30 Read block 57 len 8192 23/30 Read block 248 len 12288 24/30 Read block 431 len 28672 25/30 Read block 431 len 20480 26/30 Flush 27/30 Write block 592 len 28672 data: 6 6 5 6 5 5 4 28/30 Read block 381 len 32768 29/30 Read block 62 len 24576 30/30 Read block 431 len 4096 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:39.290137435Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1838} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 152 158 158 262 271 271 F F F <--- 2 040-059 155 158 158 268 271 271 F F F <--- 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 154 158 158 267 271 271 F F F <--- 6 120-139 155 155 155 268 268 268 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 156 156 156 269 269 269 F F F 14 280-299 156 156 156 269 269 269 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 156 158 158 269 270 270 F F F <--- 18 360-379 151 151 151 260 260 260 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 156 158 158 269 271 271 F F F <--- 23 460-479 156 156 156 269 269 269 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 156 156 156 269 269 269 F F F 26 520-539 156 156 156 269 269 269 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 146 146 146 253 253 253 F T T <--- Max gen: 158, Max flush: 271 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 158 158 158 271 271 271 F F F 2 040-059 158 158 158 271 271 271 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 155 155 155 268 268 268 F F F 7 140-159 147 147 147 255 255 255 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 155 155 268 268 268 F F F 13 260-279 156 156 156 269 269 269 F F F 14 280-299 156 156 156 269 269 269 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 151 151 151 260 260 260 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F F F 21 420-439 148 148 148 257 257 257 F F F 22 440-459 158 158 158 271 271 271 F F F 23 460-479 156 156 156 269 269 269 F F F 24 480-499 150 150 150 259 259 259 F F F 25 500-519 156 156 156 269 269 269 F F F 26 520-539 156 156 156 269 269 269 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 159 159 159 272 272 272 F F F Max gen: 159, Max flush: 272 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.60471779Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:43.615421519Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.615471235Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.625753103Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626051832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62609231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626107469Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626125127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"Crucible 1e7c76a8-900b-4fef-96c1-f5e7cb60fb82 has session id: fbf2dcda-3a7f-4159-88e3-ed2d41466616","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626475153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Upstairs opts: Upstairs UUID: 1e7c76a8-900b-4fef-96c1-f5e7cb60fb82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626506061Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Crucible stats registered with UUID: 1e7c76a8-900b-4fef-96c1-f5e7cb60fb82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.6265211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:43.626534349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626717037Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626746495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626761314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626885195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626902594Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"1e7c76a8-900b-4fef-96c1-f5e7cb60fb82 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626925783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626938582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.626952841Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62696936Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627003347Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627017996Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627032305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"ds_connection connected from Ok(127.0.0.1:56375)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627047034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"ds_connection connected from Ok(127.0.0.1:50179)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627062253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"ds_connection connected from Ok(127.0.0.1:58114)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627194444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627819262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62784766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.627864698Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628181467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628197496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[0]R flush_numbers[0..12]: [268, 271, 271, 260, 267, 271, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62828157Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[0]R generation[0..12]: [155, 158, 158, 151, 154, 158, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628298989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628315538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[1]R flush_numbers[0..12]: [268, 271, 271, 260, 267, 271, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628334406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[1]R generation[0..12]: [155, 158, 158, 151, 154, 158, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628350305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628366724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[2]R flush_numbers[0..12]: [268, 271, 271, 260, 267, 271, 268, 255, 262, 266, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628382643Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[2]R generation[0..12]: [155, 158, 158, 151, 154, 158, 155, 147, 152, 154, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628398222Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628414081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62842742Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628447659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628460938Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628474067Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"":"downstairs","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628488686Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628501645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"0","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628515794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"1","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628529733Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"client":"2","":"downstairs","session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628548622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628562111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"1e7c76a8-900b-4fef-96c1-f5e7cb60fb82 is now active with session: fbf2dcda-3a7f-4159-88e3-ed2d41466616","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.62857712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628589499Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846,"session_id":"fbf2dcda-3a7f-4159-88e3-ed2d41466616"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628602778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} {"msg":"Activated sub_volume 1e7c76a8-900b-4fef-96c1-f5e7cb60fb82","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:43.628614277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 4096 data: 13 02/30 Write block 460 len 40960 data: 18 13 12 11 10 10 13 14 14 15 03/30 Write block 138 len 12288 data: 18 15 18 04/30 Read block 198 len 12288 05/30 Read block 582 len 20480 06/30 Flush 07/30 Write block 277 len 32768 data: 10 10 10 11 13 12 9 11 08/30 Read block 15 len 32768 09/30 Flush 10/30 Write block 497 len 20480 data: 8 12 13 13 15 11/30 Read block 62 len 12288 12/30 Flush 13/30 Write block 240 len 20480 data: 10 13 16 16 13 14/30 Flush 15/30 Write block 180 len 24576 data: 16 17 16 14 13 12 16/30 Write block 477 len 24576 data: 11 15 15 17 15 14 17/30 Read block 464 len 32768 18/30 Read block 531 len 12288 19/30 Read block 277 len 16384 20/30 Read block 211 len 40960 21/30 Read block 205 len 40960 22/30 Read block 558 len 24576 23/30 Write block 21 len 32768 data: 12 8 9 13 11 12 13 11 24/30 Read block 213 len 32768 25/30 Read block 65 len 32768 26/30 Read block 106 len 12288 27/30 Read block 146 len 36864 28/30 Write block 589 len 24576 data: 6 5 5 7 7 6 29/30 Write block 371 len 24576 data: 16 14 14 14 17 17 30/30 Read block 122 len 12288 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:43.641780268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1846} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 158 158 158 271 271 271 F T T <--- 2 040-059 158 158 158 271 271 271 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 155 160 160 268 273 273 F F F <--- 7 140-159 147 160 160 255 273 273 F F F <--- 8 160-179 152 152 152 262 262 262 F F F 9 180-199 154 154 154 266 266 266 F T T <--- 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 155 160 160 268 276 276 F F F <--- 13 260-279 156 160 160 269 274 274 F F F <--- 14 280-299 156 160 160 269 274 274 F F F <--- 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 151 151 151 260 260 260 F T T <--- 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F F F 21 420-439 148 160 160 257 273 273 T F F <--- 22 440-459 158 158 158 271 271 271 F F F 23 460-479 156 160 160 269 273 273 T T T <--- 24 480-499 150 160 160 259 275 275 F T T <--- 25 500-519 156 160 160 269 275 275 F F F <--- 26 520-539 156 156 156 269 269 269 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 159 159 159 272 272 272 F T T <--- Max gen: 160, Max flush: 276 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 161 161 161 277 277 277 F F F 2 040-059 158 158 158 271 271 271 F F F 3 060-079 151 151 151 260 260 260 F F F 4 080-099 154 154 154 267 267 267 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 160 160 160 273 273 273 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 161 161 161 277 277 277 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F F F 12 240-259 160 160 160 276 276 276 F F F 13 260-279 160 160 160 274 274 274 F F F 14 280-299 160 160 160 274 274 274 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 161 161 161 277 277 277 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F F F 21 420-439 160 160 160 273 273 273 F F F 22 440-459 158 158 158 271 271 271 F F F 23 460-479 161 161 161 277 277 277 F F F 24 480-499 161 161 161 277 277 277 F F F 25 500-519 160 160 160 275 275 275 F F F 26 520-539 156 156 156 269 269 269 F F F 27 540-559 153 153 153 263 263 263 F F F 28 560-579 155 155 155 268 268 268 F F F 29 580-599 161 161 161 277 277 277 F F F Max gen: 161, Max flush: 277 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.975321816Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:47.985985718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.986038034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.996334471Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.996626701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.996664048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.996676097Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.996693976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"Crucible 288ac2f4-b8a5-4c57-a9e4-18f1df63603e has session id: 8bcd608a-089d-4f41-9824-3b9868bc5aeb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997039982Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Upstairs opts: Upstairs UUID: 288ac2f4-b8a5-4c57-a9e4-18f1df63603e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.99707476Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Crucible stats registered with UUID: 288ac2f4-b8a5-4c57-a9e4-18f1df63603e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997092789Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:47.997106098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997282796Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997320333Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997336142Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997438725Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997455724Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"288ac2f4-b8a5-4c57-a9e4-18f1df63603e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997473723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997486762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997501031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.99751527Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997549328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997569976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997584665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62050)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997599674Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58071)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997617133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62429)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.997632332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998218422Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998297806Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998464145Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998622314Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998659722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[0]R flush_numbers[0..12]: [268, 277, 271, 260, 267, 271, 273, 273, 262, 277, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.99882375Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[0]R generation[0..12]: [155, 161, 158, 151, 154, 158, 160, 160, 152, 161, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998841079Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998857858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[1]R flush_numbers[0..12]: [268, 277, 271, 260, 267, 271, 273, 273, 262, 277, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998873767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[1]R generation[0..12]: [155, 161, 158, 151, 154, 158, 160, 160, 152, 161, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998889536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998905865Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[2]R flush_numbers[0..12]: [268, 277, 271, 260, 267, 271, 273, 273, 262, 277, 269, 268]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998921554Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[2]R generation[0..12]: [155, 161, 158, 151, 154, 158, 160, 160, 152, 161, 156, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998940253Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998956971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3"Wait for a query_work_queue command to finish before sending IO ,"pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.99897714Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.998991749Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999005288Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999018627Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"":"downstairs","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999032826Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999045515Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"0","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999059624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"1","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999076423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"client":"2","":"downstairs","session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999091262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999104801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"288ac2f4-b8a5-4c57-a9e4-18f1df63603e is now active with session: 8bcd608a-089d-4f41-9824-3b9868bc5aeb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.99912008Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999132889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854,"session_id":"8bcd608a-089d-4f41-9824-3b9868bc5aeb"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999146318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} {"msg":"Activated sub_volume 288ac2f4-b8a5-4c57-a9e4-18f1df63603e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:47.999157838Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 477 len 28672 data: 12 16 16 18 16 15 13 02/30 Write block 553 len 12288 data: 10 11 12 03/30 Write block 26 len 12288 data: 13 14 12 04/30 Flush 05/30 Write block 552 len 4096 data: 8 06/30 Write block 435 len 8192 data: 12 11 07/30 Write block 578 len 40960 data: 8 9 8 11 11 12 11 9 9 6 08/30 Write block 66 len 16384 data: 17 18 20 19 09/30 Read block 84 len 28672 10/30 Write block 536 len 20480 data: 9 13 11 10 11 11/30 Read block 486 len 12288 12/30 Read block 315 len 12288 13/30 Flush 14/30 Write block 510 len 16384 data: 14 12 11 9 15/30 Flush 16/30 Write block 90 len 8192 data: 8 9 17/30 Write block 416 len 40960 data: 12 11 9 10 9 10 9 10 12 12 18/30 Read block 310 len 28672 19/30 Read block 326 len 16384 20/30 Read block 74 len 36864 21/30 Read block 103 len 4096 22/30 Write block 496 len 28672 data: 10 9 13 14 14 16 19 23/30 Read block 576 len 20480 24/30 Read block 112 len 24576 25/30 Read block 330 len 40960 26/30 Write block 429 len 12288 data: 17 16 15 27/30 Read block 9 len 4096 28/30 Write block 238 len 12288 data: 9 11 11 29/30 Write block 429 len 24576 data: 18 17 16 14 11 12 30/30 Read block 275 len 4096 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 2 31 Done 1 30 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:48.011271Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1854} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 161 162 162 277 278 278 F F F <--- 2 040-059 158 158 158 271 271 271 F F F 3 060-079 151 162 162 260 279 279 F F F <--- 4 080-099 154 154 154 267 267 267 F T T <--- 5 100-119 158 158 158 271 271 271 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 160 160 160 273 273 273 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 161 161 161 277 277 277 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 155 155 155 268 268 268 F T T <--- 12 240-259 160 160 160 276 276 276 F T T <--- 13 260-279 160 160 160 274 274 274 F F F 14 280-299 160 160 160 274 274 274 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 161 161 161 277 277 277 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 156 156 156 269 269 269 F T T <--- 21 420-439 160 162 162 273 279 279 F T T <--- 22 440-459 158 158 158 271 271 271 F F F 23 460-479 161 162 162 277 278 278 T F F <--- 24 480-499 161 162 162 277 278 278 T T T <--- 25 500-519 160 162 162 275 280 280 F T T <--- 26 520-539 156 162 162 269 279 279 F F F <--- 27 540-559 153 162 162 263 279 279 F F F <--- 28 560-579 155 162 162 268 279 279 F F F <--- 29 580-599 161 162 162 277 279 279 F F F <--- Max gen: 162, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 162 162 162 278 278 278 F F F 2 040-059 158 158 158 271 271 271 F F F 3 060-079 162 162 162 279 279 279 F F F 4 080-099 163 163 163 281 281 281 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 160 160 160 273 273 273 F F F 8 160-179 152 152 152 262 262 262 F F F 9 180-199 161 161 161 277 277 277 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 163 163 163 281 281 281 F F F 12 240-259 163 163 163 281 281 281 F F F 13 260-279 160 160 160 274 274 274 F F F 14 280-299 160 160 160 274 274 274 F F F 15 300-319 152 152 152 262 262 262 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 161 161 161 277 277 277 F F F 19 380-399 156 156 156 269 269 269 F F F 20 400-419 163 163 163 281 281 281 F F F 21 420-439 163 163 163 281 281 281 F F F 22 440-459 158 158 158 271 271 271 F F F 23 460-479 162 162 162 278 278 278 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 163 163 163 281 281 281 F F F 26 520-539 162 162 162 279 279 279 F F F 27 540-559 162 162 162 279 279 279 F F F 28 560-579 162 162 162 279 279 279 F F F 29 580-599 162 162 162 279 279 279 F F F Max gen: 163, Max flush: 281 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.351226431Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363123798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363440466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363482784Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363495313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363513191Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"Crucible 6c73fe4e-78ec-430e-8a4e-8f8c0360841f has session id: c0317834-44d7-471c-940d-6e427eea48a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36383121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Upstairs opts: Upstairs UUID: 6c73fe4e-78ec-430e-8a4e-8f8c0360841f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363858148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Crucible stats registered with UUID: 6c73fe4e-78ec-430e-8a4e-8f8c0360841f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.363873027Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:52.363885956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364078503Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36411578Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364131279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364246201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36426328Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"6c73fe4e-78ec-430e-8a4e-8f8c0360841f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364279459Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364292258Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364306227Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364320336Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364333945Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364348084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364362203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39592)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364422989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52328)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364572319Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43974)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.364599477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365063885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365088584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365325828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36543426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365474717Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[0]R flush_numbers[0..12]: [268, 278, 271, 279, 281, 271, 273, 273, 262, 277, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365680633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[0]R generation[0..12]: [155, 162, 158, 162, 163, 158, 160, 160, 152, 161, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365697792Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365714431Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[1]R flush_numbers[0..12]: [268, 278, 271, 279, 281, 271, 273, 273, 262, 277, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36573077Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[1]R generation[0..12]: [155, 162, 158, 162, 163, 158, 160, 160, 152, 161, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365746419Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365762538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[2]R flush_numbers[0..12]: [268, 278, 271, 279, 281, 271, 273, 273, 262, 277, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365778557Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[2]R generation[0..12]: [155, 162, 158, 162, 163, 158, 160, 160, 152, 161, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365794126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365810274Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:37:52.365823624Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365842662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365855961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.36586949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"":"downstairs","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365883339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365897948Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"0","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365912208Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"1","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365926097Z","hostname":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"client":"2","":"downstairs","session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365943375Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365956814Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"6c73fe4e-78ec-430e-8a4e-8f8c0360841f is now active with session: c0317834-44d7-471c-940d-6e427eea48a1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365971493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365983973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862,"session_id":"c0317834-44d7-471c-940d-6e427eea48a1"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.365997322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} {"msg":"Activated sub_volume 6c73fe4e-78ec-430e-8a4e-8f8c0360841f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:52.366008731Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 28672 data: 11 11 9 9 11 11 10 02/30 Read block 517 len 16384 03/30 Read block 514 len 20480 04/30 Write block 417 len 12288 data: 12 10 11 05/30 Read block 22 len 28672 06/30 Write block 94 len 16384 data: 10 12 16 16 07/30 Read block 38 len 28672 08/30 Read block 139 len 36864 09/30 Read block 259 len 20480 10/30 Read block 301 len 8192 11/30 Write block 566 len 24576 data: 11 9 9 6 10 10 12/30 Read block 58 len 4096 13/30 Read block 324 len 20480 14/30 Read block 180 len 8192 15/30 Read block 272 len 40960 16/30 Write block 372 len 24576 data: 15 15 15 18 18 15 17/30 Flush 18/30 Write block 534 len 40960 data: 12 9 10 14 12 11 12 14 15 12 19/30 Write block 393 len 8192 data: 15 15 20/30 Write block 288 len 8192 data: 12 10 21/30 Read block 155 len 4096 22/30 Read block 410 len 20480 23/30 Flush 24/30 Write block 584 len 32768 data: 12 10 10 7 5 7 6 6 25/30 Write block 176 len 20480 data: 11 11 12 15 17 26/30 Read block 554 len 36864 27/30 Write block 79 len 12288 data: 11 10 13 28/30 Read block 594 len 24576 29/30 Read block 276 len 16384 30/30 Read block 354 len 8192 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 4 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:52.378133452Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1862} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 162 162 162 278 278 278 F F F 2 040-059 158 158 158 271 271 271 F F F 3 060-079 162 162 162 279 279 279 T T F <--- 4 080-099 164 164 163 282 282 281 T T F <--- 5 100-119 158 158 158 271 271 271 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 160 160 160 273 273 273 F F F 8 160-179 152 152 152 262 262 262 T T F <--- 9 180-199 161 161 161 277 277 277 T T F <--- 10 200-219 156 156 156 269 269 269 F F F 11 220-239 163 163 163 281 281 281 F F F 12 240-259 163 163 163 281 281 281 F F F 13 260-279 160 160 160 274 274 274 F F F 14 280-299 164 164 160 283 283 274 F F F <--- 15 300-319 164 164 152 282 282 262 F F F <--- 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 164 164 161 282 282 277 F F F <--- 19 380-399 164 164 156 283 283 269 F F F <--- 20 400-419 164 164 163 282 282 281 F F F <--- 21 420-439 163 163 163 281 281 281 F F F 22 440-459 158 158 158 271 271 271 F F F 23 460-479 162 162 162 278 278 278 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 163 163 163 281 281 281 F F F 26 520-539 164 164 162 283 283 279 F F F <--- 27 540-559 164 164 162 283 283 279 F F F <--- 28 560-579 164 164 162 282 282 279 F F F <--- 29 580-599 162 162 162 279 279 279 T T F <--- Max gen: 164, Max flush: 283 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 F F F 1 020-039 162 162 162 278 278 278 F F F 2 040-059 158 158 158 271 271 271 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 160 160 160 273 273 273 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 156 156 156 269 269 269 F F F 11 220-239 163 163 163 281 281 281 F F F 12 240-259 163 163 163 281 281 281 F F F 13 260-279 160 160 160 274 274 274 F F F 14 280-299 164 164 164 283 283 283 F F F 15 300-319 164 164 164 282 282 282 F F F 16 320-339 153 153 153 263 263 263 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 164 164 164 282 282 282 F F F 19 380-399 164 164 164 283 283 283 F F F 20 400-419 164 164 164 282 282 282 F F F 21 420-439 163 163 163 281 281 281 F F F 22 440-459 158 158 158 271 271 271 F F F 23 460-479 162 162 162 278 278 278 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 163 163 163 281 281 281 F F F 26 520-539 164 164 164 283 283 283 F F F 27 540-559 164 164 164 283 283 283 F F F 28 560-579 164 164 164 282 282 282 F F F 29 580-599 165 165 165 284 284 284 F F F Max gen: 165, Max flush: 284 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.704839429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.71653579Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.716856628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.716895025Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.716907534Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.716925173Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"Crucible a4452b25-976c-4189-8ac2-0a43f5e9bf9f has session id: d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717304177Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Upstairs opts: Upstairs UUID: a4452b25-976c-4189-8ac2-0a43f5e9bf9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717342404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Crucible stats registered with UUID: a4452b25-976c-4189-8ac2-0a43f5e9bf9f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717358403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:56.717371892Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717530982Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.71755701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717572149Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717673992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.71769305Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"a4452b25-976c-4189-8ac2-0a43f5e9bf9f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717723348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717736447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717750427Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717764206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717777835Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717792084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717806433Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34669)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717893487Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55762)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717909896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58571)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.717925065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.718703401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.718783206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.718801735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719112783Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719144371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[0]R flush_numbers[0..12]: [268, 278, 271, 284, 284, 271, 273, 273, 284, 284, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719213627Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[0]R generation[0..12]: [155, 162, 158, 165, 165, 158, 160, 160, 165, 165, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719231265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719247984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[1]R flush_numbers[0..12]: [268, 278, 271, 284, 284, 271, 273, 273, 284, 284, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719264193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[1]R generation[0..12]: [155, 162, 158, 165, 165, 158, 160, 160, 165, 165, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719280122Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719296461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[2]R flush_numbers[0..12]: [268, 278, 271, 284, 284, 271, 273, 273, 284, 284, 269, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.71931565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[2]R generation[0..12]: [155, 162, 158, 165, 165, 158, 160, 160, 165, 165, 156, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719331708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719348277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Max found gen is 166","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-03-16T21:37:56.719368006Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719382545Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719396184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719409483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"":"downstairs","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719423252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719435841Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"0","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.71944986Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"1","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719463929Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"client":"2","":"downstairs","session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719477798Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719493637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"a4452b25-976c-4189-8ac2-0a43f5e9bf9f is now active with session: d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719508606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719521426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870,"session_id":"d46d3efa-3a2e-4d4e-91cb-6ff2c7b0a99c"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719536984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} {"msg":"Activated sub_volume a4452b25-976c-4189-8ac2-0a43f5e9bf9f","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:37:56.719548534Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 255 len 28672 data: 6 6 9 9 9 9 12 02/30 Write block 135 len 4096 data: 12 03/30 Write block 136 len 32768 data: 16 16 19 16 19 17 14 11 04/30 Read block 394 len 8192 05/30 Read block 175 len 40960 06/30 Read block 7 len 36864 07/30 Flush 08/30 Flush 09/30 Flush 10/30 Write block 373 len 12288 data: 16 16 19 11/30 Write block 218 len 8192 data: 13 15 12/30 Read block 521 len 4096 13/30 Read block 423 len 40960 14/30 Write block 512 len 4096 data: 12 15/30 Write block 325 len 20480 data: 8 9 10 9 11 16/30 Write block 21 len 28672 data: 13 9 10 14 12 14 15 17/30 Flush 18/30 Flush 19/30 Write block 522 len 8192 data: 11 7 20/30 Write block 575 len 16384 data: 11 7 6 9 21/30 Write block 272 len 20480 data: 13 11 11 11 8 22/30 Write block 11 len 16384 data: 15 17 16 16 23/30 Write block 435 len 32768 data: 13 12 13 13 9 8 11 11 24/30 Write block 521 len 40960 data: 12 12 8 7 6 8 11 13 13 13 25/30 Read block 475 len 28672 26/30 Write block 35 len 24576 data: 14 14 15 19 19 18 27/30 Write block 35 len 12288 data: 15 15 16 28/30 Read block 394 len 24576 29/30 Write block 126 len 24576 data: 14 13 12 12 11 11 30/30 Read block 201 len 28672 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 2 29 Done 30 3 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:37:56.730388493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1870} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 268 268 268 T F T <--- 1 020-039 166 162 166 288 278 288 T F T <--- 2 040-059 158 158 158 271 271 271 T F T <--- 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 166 160 166 285 273 285 T T T <--- 7 140-159 166 160 166 285 273 285 F T F <--- 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 156 166 288 269 288 F F F <--- 11 220-239 163 163 163 281 281 281 F F F 12 240-259 166 163 166 285 281 285 F T F <--- 13 260-279 166 160 166 285 274 285 T T T <--- 14 280-299 164 164 164 283 283 283 F F F 15 300-319 164 164 164 282 282 282 F F F 16 320-339 166 153 166 288 263 288 F F F <--- 17 340-359 158 158 158 270 270 270 F F F 18 360-379 166 164 166 288 282 288 F F F <--- 19 380-399 164 164 164 283 283 283 F F F 20 400-419 164 164 164 282 282 282 F F F 21 420-439 163 163 163 281 281 281 T F T <--- 22 440-459 158 158 158 271 271 271 T F T <--- 23 460-479 162 162 162 278 278 278 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 166 163 166 288 281 288 F F F <--- 26 520-539 164 164 164 283 283 283 T F T <--- 27 540-559 164 164 164 283 283 283 F F F 28 560-579 164 164 164 282 282 282 T F T <--- 29 580-599 165 165 165 284 284 284 F F F Max gen: 166, Max flush: 288 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 289 289 289 F F F 1 020-039 167 167 167 289 289 289 F F F 2 040-059 167 167 167 289 289 289 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 166 166 288 288 288 F F F 11 220-239 163 163 163 281 281 281 F F F 12 240-259 166 166 166 285 285 285 F F F 13 260-279 167 167 167 289 289 289 F F F 14 280-299 164 164 164 283 283 283 F F F 15 300-319 164 164 164 282 282 282 F F F 16 320-339 166 166 166 288 288 288 F F F 17 340-359 158 158 158 270 270 270 F F F 18 360-379 166 166 166 288 288 288 F F F 19 380-399 164 164 164 283 283 283 F F F 20 400-419 164 164 164 282 282 282 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 162 162 162 278 278 278 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 166 166 166 288 288 288 F F F 26 520-539 167 167 167 289 289 289 F F F 27 540-559 164 164 164 283 283 283 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 165 165 165 284 284 284 F F F Max gen: 167, Max flush: 289 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.06233859Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:01.073018141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.073072677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.083581599Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.083919396Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.083959413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.083971482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.083989161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"Crucible cc5050e5-4fc6-43c0-a2f5-0d615186651e has session id: 249b997a-1dbc-48d0-9bba-fb0caffb4206","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084357576Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Upstairs opts: Upstairs UUID: cc5050e5-4fc6-43c0-a2f5-0d615186651e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084392304Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Crucible stats registered with UUID: cc5050e5-4fc6-43c0-a2f5-0d615186651e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084408032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:01.084421692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084615828Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084647936Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084663215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084781607Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084799636Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"cc5050e5-4fc6-43c0-a2f5-0d615186651e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084816395Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084829234Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084843433Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084857392Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084871201Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.08488532Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084899729Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"ds_connection connected from Ok(127.0.0.1:46987)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084914498Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"ds_connection connected from Ok(127.0.0.1:53416)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084935636Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"ds_connection connected from Ok(127.0.0.1:40461)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.084950635Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.085730482Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.08576763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.085787528Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086109166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086156203Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[0]R flush_numbers[0..12]: [289, 289, 289, 284, 284, 271, 289, 285, 284, 284, 288, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086219939Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[0]R generation[0..12]: [167, 167, 167, 165, 165, 158, 167, 166, 165, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086236918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086253656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[1]R flush_numbers[0..12]: [289, 289, 289, 284, 284, 271, 289, 285, 284, 284, 288, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086269465Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[1]R generation[0..12]: [167, 167, 167, 165, 165, 158, 167, 166, 165, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086285444Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086301693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[2]R flush_numbers[0..12]: [289, 289, 289, 284, 284, 271, 289, 285, 284, 284, 288, 281]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086317362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[2]R generation[0..12]: [167, 167, 167, 165, 165, 158, 167, 166, 165, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086332711Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.08634883Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086362389Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086384917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086398426Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086411886Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"":"downstairs","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086426125Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086438774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"0","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086452933Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"1","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:38:01.086466822Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"client":"2","":"downstairs","session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086483881Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.08649693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"cc5050e5-4fc6-43c0-a2f5-0d615186651e is now active with session: 249b997a-1dbc-48d0-9bba-fb0caffb4206","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086511949Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086524858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878,"session_id":"249b997a-1dbc-48d0-9bba-fb0caffb4206"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086538207Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} {"msg":"Activated sub_volume cc5050e5-4fc6-43c0-a2f5-0d615186651e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:01.086549746Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 28672 data: 17 16 14 12 12 13 11 02/30 Read block 293 len 4096 03/30 Write block 542 len 8192 data: 16 13 04/30 Write block 393 len 24576 data: 16 16 16 20 20 21 05/30 Read block 365 len 28672 06/30 Flush 07/30 Write block 291 len 4096 data: 10 08/30 Flush 09/30 Write block 310 len 36864 data: 8 8 9 15 15 16 16 18 17 10/30 Write block 129 len 16384 data: 13 12 12 13 11/30 Write block 13 len 40960 data: 17 17 15 13 16 15 14 12 14 10 12/30 Write block 517 len 28672 data: 17 17 14 14 13 13 9 13/30 Read block 462 len 28672 14/30 Read block 547 len 20480 15/30 Read block 270 len 32768 16/30 Write block 349 len 8192 data: 12 11 17/30 Read block 375 len 36864 18/30 Read block 15 len 28672 19/30 Read block 435 len 12288 20/30 Read block 228 len 4096 21/30 Write block 393 len 8192 data: 17 17 22/30 Write block 241 len 32768 data: 14 17 17 14 13 16 11 13 23/30 Write block 342 len 8192 data: 11 10 24/30 Write block 464 len 16384 data: 11 11 14 15 25/30 Write block 234 len 16384 data: 18 16 15 11 26/30 Read block 307 len 16384 27/30 Write block 525 len 24576 data: 7 9 12 14 14 14 28/30 Write block 543 len 8192 data: 14 11 29/30 Write block 337 len 24576 data: 14 14 12 10 12 12 30/30 Read block 596 len 8192 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:01.097576703Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1878} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 289 289 289 F T T <--- 1 020-039 167 167 167 289 289 289 F T T <--- 2 040-059 167 167 167 289 289 289 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 167 167 167 289 289 289 F T T <--- 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 166 166 288 288 288 F F F 11 220-239 163 163 163 281 281 281 F T T <--- 12 240-259 166 166 166 285 285 285 F T T <--- 13 260-279 167 167 167 289 289 289 F F F 14 280-299 164 168 168 283 291 291 F F F <--- 15 300-319 164 168 168 282 290 290 T T T <--- 16 320-339 166 168 168 288 290 290 T T T <--- 17 340-359 158 158 158 270 270 270 F T T <--- 18 360-379 166 166 166 288 288 288 F F F 19 380-399 164 168 168 283 290 290 T T T <--- 20 400-419 164 164 164 282 282 282 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 162 162 162 278 278 278 F T T <--- 24 480-499 163 163 163 281 281 281 F F F 25 500-519 166 166 166 288 288 288 F T T <--- 26 520-539 167 167 167 289 289 289 F T T <--- 27 540-559 164 168 168 283 290 290 T T T <--- 28 560-579 167 167 167 289 289 289 F F F 29 580-599 165 165 165 284 284 284 F F F Max gen: 168, Max flush: 291 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 292 292 292 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 167 167 167 289 289 289 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 166 166 288 288 288 F F F 11 220-239 169 169 169 292 292 292 F F F 12 240-259 169 169 169 292 292 292 F F F 13 260-279 167 167 167 289 289 289 F F F 14 280-299 168 168 168 291 291 291 F F F 15 300-319 169 169 169 292 292 292 F F F 16 320-339 169 169 169 292 292 292 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 166 166 166 288 288 288 F F F 19 380-399 169 169 169 292 292 292 F F F 20 400-419 164 164 164 282 282 282 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 169 169 169 292 292 292 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 169 169 169 292 292 292 F F F 26 520-539 169 169 169 292 292 292 F F F 27 540-559 169 169 169 292 292 292 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 165 165 165 284 284 284 F F F Max gen: 169, Max flush: 292 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.440748124Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.452721446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453057233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453095131Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.45311023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453128388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"Crucible 497c179a-7202-4e17-8440-1053b30505c5 has session id: 8c1788e0-b0fd-4a02-97a2-5e61acd9882c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453434497Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Upstairs opts: Upstairs UUID: 497c179a-7202-4e17-8440-1053b30505c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453464955Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Crucible stats registered with UUID: 497c179a-7202-4e17-8440-1053b30505c5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453480224Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:05.453493763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453676411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453700609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453715288Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.45383409Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453852759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"497c179a-7202-4e17-8440-1053b30505c5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453867548Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453880027Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453893796Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453909575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453923524Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453937673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.453952192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46227)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454040836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65415)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454164547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57316)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454192136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454765296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454826282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.454844521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455176428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455212186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[0]R flush_numbers[0..12]: [292, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455281071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[0]R generation[0..12]: [169, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.45529926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455316229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[1]R flush_numbers[0..12]: [292, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455335277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[1]R generation[0..12]: [169, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455351146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455367315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[2]R flush_numbers[0..12]: [292, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455383084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[2]R generation[0..12]: [169, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455398673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455414972Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455428371Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":Wait for a query_work_queue command to finish before sending IO 1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.45544917Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455462369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455475888Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"":"downstairs","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455489937Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455502466Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"0","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455516565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"1","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:38:05.455543193Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"client":"2","":"downstairs","session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455562512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455575981Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"497c179a-7202-4e17-8440-1053b30505c5 is now active with session: 8c1788e0-b0fd-4a02-97a2-5e61acd9882c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.45559076Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455603699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886,"session_id":"8c1788e0-b0fd-4a02-97a2-5e61acd9882c"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455616918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} {"msg":"Activated sub_volume 497c179a-7202-4e17-8440-1053b30505c5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:05.455628267Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 4096 data: 15 02/30 Read block 90 len 12288 03/30 Read block 438 len 32768 04/30 Read block 222 len 16384 05/30 Read block 190 len 32768 06/30 Read block 383 len 24576 07/30 Write block 311 len 28672 data: 9 10 16 16 17 17 19 08/30 Flush 09/30 Write block 474 len 8192 data: 10 7 10/30 Read block 144 len 8192 11/30 Flush 12/30 Write block 316 len 28672 data: 18 20 18 16 13 13 14 13/30 Write block 399 len 8192 data: 18 19 14/30 Write block 432 len 28672 data: 15 12 13 14 13 14 14 15/30 Write block 587 len 20480 data: 8 6 8 7 7 16/30 Read block 441 len 12288 17/30 Read block 515 len 40960 18/30 Write block 518 len 36864 data: 18 15 15 14 14 10 8 8 10 19/30 Read block 282 len 40960 20/30 Write block 396 len 28672 data: 21 21 22 19 20 15 17 21/30 Read block 271 len 32768 22/30 Write block 241 len 8192 data: 15 18 23/30 Read block 374 len 8192 24/30 Read block 216 len 16384 25/30 Flush 26/30 Write block 320 len 8192 data: 14 14 27/30 Write block 3 len 36864 data: 10 9 11 13 13 12 16 16 16 28/30 Write block 316 len 12288 data: 19 21 19 29/30 Write block 372 len 24576 data: 16 17 17 20 19 16 30/30 Write block 254 len 12288 data: 8 7 7 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 26 4 34 Done 26 4 26 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:05.467018979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1886} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 292 292 292 T F T <--- 1 020-039 169 169 169 292 292 292 F F F 2 040-059 167 167 167 289 289 289 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 166 166 288 288 288 F F F 11 220-239 169 169 169 292 292 292 F F F 12 240-259 170 169 170 295 292 295 T F T <--- 13 260-279 167 167 167 289 289 289 F F F 14 280-299 168 168 168 291 291 291 F F F 15 300-319 170 169 170 295 292 295 T T T <--- 16 320-339 170 169 170 295 292 295 T F T <--- 17 340-359 169 169 169 292 292 292 F F F 18 360-379 166 166 166 288 288 288 T F T <--- 19 380-399 170 169 170 295 292 295 F F F <--- 20 400-419 170 164 170 295 282 295 F F F <--- 21 420-439 170 167 170 295 289 295 F F F <--- 22 440-459 167 167 167 289 289 289 F F F 23 460-479 170 169 170 294 292 294 F F F <--- 24 480-499 163 163 163 281 281 281 F F F 25 500-519 170 169 170 295 292 295 F F F <--- 26 520-539 170 169 170 295 292 295 F F F <--- 27 540-559 169 169 169 292 292 292 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 170 165 170 295 284 295 F F F <--- Max gen: 170, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 167 167 167 289 289 289 F F F 3 060-079 165 165 165 284 284 284 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F F F 9 180-199 165 165 165 284 284 284 F F F 10 200-219 166 166 166 288 288 288 F F F 11 220-239 169 169 169 292 292 292 F F F 12 240-259 171 171 171 296 296 296 F F F 13 260-279 167 167 167 289 289 289 F F F 14 280-299 168 168 168 291 291 291 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 171 171 171 296 296 296 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 171 171 171 296 296 296 F F F 19 380-399 170 170 170 295 295 295 F F F 20 400-419 170 170 170 295 295 295 F F F 21 420-439 170 170 170 295 295 295 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 170 170 170 294 294 294 F F F 24 480-499 163 163 163 281 281 281 F F F 25 500-519 170 170 170 295 295 295 F F F 26 520-539 170 170 170 295 295 295 F F F 27 540-559 169 169 169 292 292 292 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 170 170 170 295 295 295 F F F Max gen: 171, Max flush: 296 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.792842856Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:09.803543844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.80359456Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814021268Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814322407Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814359385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814371424Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814389363Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"Crucible 894ebb42-fcdd-413b-a724-5edaf259cc0c has session id: 91053808-ca53-4204-9951-794e4ea235b1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814755108Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Upstairs opts: Upstairs UUID: 894ebb42-fcdd-413b-a724-5edaf259cc0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814790245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Crucible stats registered with UUID: 894ebb42-fcdd-413b-a724-5edaf259cc0c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814808824Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:09.814822553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.814990902Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.81501566Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815030349Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.81516307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815181479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"894ebb42-fcdd-413b-a724-5edaf259cc0c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815202047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815214856Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815228715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815242584Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815256164Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815270763Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815285252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61155)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815328549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37276)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815346667Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61636)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.815361546Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816107255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816143413Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816160192Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816486619Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816531526Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[0]R flush_numbers[0..12]: [296, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816609751Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[0]R generation[0..12]: [171, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.81662694Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816643669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[1]R flush_numbers[0..12]: [296, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816659638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[1]R generation[0..12]: [171, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816681776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816698215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[2]R flush_numbers[0..12]: [296, 292, 289, 284, 284, 271, 292, 285, 284, 284, 288, 292]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816713734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[2]R generation[0..12]: [171, 169, 167, 165, 165, 158, 169, 166, 165, 165, 166, 169]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816732353Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816748722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816762361Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:38:09.81677645Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816795848Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816809047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"":"downstairs","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816823436Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816836136Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"0","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816850255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"1","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816864354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"client":"2","":"downstairs","session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816878973Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816894292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"894ebb42-fcdd-413b-a724-5edaf259cc0c is now active with session: 91053808-ca53-4204-9951-794e4ea235b1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816909411Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.81692213Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894,"session_id":"91053808-ca53-4204-9951-794e4ea235b1"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816935669Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} {"msg":"Activated sub_volume 894ebb42-fcdd-413b-a724-5edaf259cc0c","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:09.816947408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 498 len 20480 data: 14 15 15 17 20 02/30 Read block 488 len 16384 03/30 Flush 04/30 Write block 280 len 40960 data: 12 14 13 10 12 10 9 11 13 11 05/30 Read block 47 len 20480 06/30 Write block 533 len 24576 data: 12 13 10 11 15 13 07/30 Read block 184 len 12288 08/30 Read block 432 len 24576 09/30 Write block 527 len 32768 data: 13 15 15 15 12 11 13 14 10/30 Write block 264 len 24576 data: 12 10 11 10 13 14 11/30 Write block 543 len 32768 data: 15 12 9 8 9 10 10 8 12/30 Read block 27 len 20480 13/30 Read block 318 len 40960 14/30 Read block 376 len 24576 15/30 Write block 181 len 4096 data: 18 16/30 Read block 285 len 36864 17/30 Read block 211 len 32768 18/30 Read block 59 len 24576 19/30 Read block 89 len 8192 20/30 Flush 21/30 Write block 170 len 16384 data: 14 11 13 12 22/30 Read block 132 len 8192 23/30 Write block 215 len 4096 data: 11 24/30 Write block 41 len 40960 data: 14 13 12 12 12 13 11 12 13 12 25/30 Read block 115 len 28672 26/30 Write block 227 len 28672 data: 21 19 17 20 18 18 17 27/30 Write block 71 len 12288 data: 17 15 17 28/30 Read block 127 len 40960 29/30 Write block 536 len 28672 data: 12 16 14 12 13 15 17 30/30 Write block 232 len 12288 data: 19 18 19 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:09.829781101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1894} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 167 167 167 289 289 289 F T T <--- 3 060-079 165 165 165 284 284 284 F T T <--- 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 165 165 165 284 284 284 F T T <--- 9 180-199 165 172 172 284 298 298 F F F <--- 10 200-219 166 166 166 288 288 288 F T T <--- 11 220-239 169 169 169 292 292 292 F T T <--- 12 240-259 171 171 171 296 296 296 F F F 13 260-279 167 172 172 289 298 298 F F F <--- 14 280-299 168 172 172 291 298 298 F F F <--- 15 300-319 171 171 171 296 296 296 F F F 16 320-339 171 171 171 296 296 296 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 171 171 171 296 296 296 F F F 19 380-399 170 170 170 295 295 295 F F F 20 400-419 170 170 170 295 295 295 F F F 21 420-439 170 170 170 295 295 295 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 170 170 170 294 294 294 F F F 24 480-499 163 172 172 281 297 297 T F F <--- 25 500-519 170 172 172 295 297 297 T F F <--- 26 520-539 170 172 172 295 298 298 F T T <--- 27 540-559 169 172 172 292 298 298 F T T <--- 28 560-579 167 167 167 289 289 289 F F F 29 580-599 170 170 170 295 295 295 F F F Max gen: 172, Max flush: 298 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 173 173 173 299 299 299 F F F 3 060-079 173 173 173 299 299 299 F F F 4 080-099 165 165 165 284 284 284 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 F F F 11 220-239 173 173 173 299 299 299 F F F 12 240-259 171 171 171 296 296 296 F F F 13 260-279 172 172 172 298 298 298 F F F 14 280-299 172 172 172 298 298 298 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 171 171 171 296 296 296 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 171 171 171 296 296 296 F F F 19 380-399 170 170 170 295 295 295 F F F 20 400-419 170 170 170 295 295 295 F F F 21 420-439 170 170 170 295 295 295 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 170 170 170 294 294 294 F F F 24 480-499 172 172 172 297 297 297 F F F 25 500-519 172 172 172 297 297 297 F F F 26 520-539 173 173 173 299 299 299 F F F 27 540-559 173 173 173 299 299 299 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 170 170 170 295 295 295 F F F Max gen: 173, Max flush: 299 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.164281965Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:14.174985283Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.175046849Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.185309078Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.185597748Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.185636606Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.185648445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.185666114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"Crucible 7f074219-dd5c-470a-a191-8bbfb2c80612 has session id: ac4cc77e-a932-478f-8541-908213801ee9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186043028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Upstairs opts: Upstairs UUID: 7f074219-dd5c-470a-a191-8bbfb2c80612, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186077975Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Crucible stats registered with UUID: 7f074219-dd5c-470a-a191-8bbfb2c80612","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186093344Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:14.186106993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186282971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.18630928Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186327428Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186463109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186480758Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"7f074219-dd5c-470a-a191-8bbfb2c80612 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186495587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186508166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186522205Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186535924Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186549523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186574382Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186589141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58182)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186653836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36472)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186670825Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64855)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.186686044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187456051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187492079Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187509068Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187807437Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187851144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[0]R flush_numbers[0..12]: [296, 292, 299, 299, 284, 271, 292, 285, 299, 298, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187927549Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[0]R generation[0..12]: [171, 169, 173, 173, 165, 158, 169, 166, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187944918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187961467Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[1]R flush_numbers[0..12]: [296, 292, 299, 299, 284, 271, 292, 285, 299, 298, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187977226Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[1]R generation[0..12]: [171, 169, 173, 173, 165, 158, 169, 166, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.187992865Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188009223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[2]R flush_numbers[0..12]: [296, 292, 299, 299, 284, 271, 292, 285, 299, 298, 299, 299]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188024862Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[2]R generation[0..12]: [171, 169, 173, 173, 165, 158, 169, 166, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188040561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.18805692Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188070329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188090488Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188104047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188119826Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"":"downstairs","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188134385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188147184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"0","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188161633Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"1","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:38:14.188175652Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"client":"2","":"downstairs","session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188193221Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.18820691Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"7f074219-dd5c-470a-a191-8bbfb2c80612 is now active with session: ac4cc77e-a932-478f-8541-908213801ee9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188222369Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188235188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902,"session_id":"ac4cc77e-a932-478f-8541-908213801ee9"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188248617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} {"msg":"Activated sub_volume 7f074219-dd5c-470a-a191-8bbfb2c80612","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:14.188260166Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 12288 data: 17 16 13 02/30 Write block 80 len 16384 data: 11 14 10 7 03/30 Read block 62 len 36864 04/30 Write block 592 len 8192 data: 8 8 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 285 len 16384 data: 11 10 12 14 09/30 Write block 400 len 24576 data: 21 16 18 15 14 13 10/30 Read block 170 len 8192 11/30 Flush 12/30 Write block 562 len 24576 data: 12 12 14 10 12 10 13/30 Write block 479 len 28672 data: 17 19 17 16 14 10 11 14/30 Read block 164 len 4096 15/30 Read block 582 len 32768 16/30 Write block 565 len 28672 data: 11 13 11 10 7 11 11 17/30 Read block 520 len 36864 18/30 Write block 582 len 24576 data: 12 13 13 11 11 9 19/30 Read block 6 len 24576 20/30 Flush 21/30 Write block 421 len 32768 data: 11 10 11 13 13 14 14 16 22/30 Flush 23/30 Write block 516 len 36864 data: 15 18 19 16 16 15 15 11 9 24/30 Read block 236 len 16384 25/30 Flush 26/30 Write block 233 len 8192 data: 19 20 27/30 Read block 136 len 28672 28/30 Read block 299 len 20480 29/30 Write block 376 len 24576 data: 20 17 16 16 17 13 30/30 Write block 490 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 18 2 2 22 Done 1 17 17 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:14.200134795Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1902} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 173 173 173 299 299 299 F F F 3 060-079 173 173 173 299 299 299 F F F 4 080-099 174 174 174 300 300 300 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 F F F 11 220-239 173 173 173 299 299 299 F T T <--- 12 240-259 171 171 171 296 296 296 F F F 13 260-279 172 172 172 298 298 298 F F F 14 280-299 174 174 174 303 303 303 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 171 171 171 296 296 296 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 171 171 171 296 296 296 F T T <--- 19 380-399 170 170 170 295 295 295 F T T <--- 20 400-419 174 174 174 303 303 303 F F F 21 420-439 174 174 174 300 305 305 F F F <--- 22 440-459 167 167 167 289 289 289 F F F 23 460-479 170 174 174 294 304 304 F F F <--- 24 480-499 172 174 174 297 304 304 F T T <--- 25 500-519 172 174 174 297 306 306 F F F <--- 26 520-539 173 174 174 299 306 306 F F F <--- 27 540-559 173 173 173 299 299 299 F F F 28 560-579 167 174 174 289 304 304 T F F <--- 29 580-599 174 174 174 300 304 304 F F F <--- Max gen: 174, Max flush: 306 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 169 169 169 292 292 292 F F F 2 040-059 173 173 173 299 299 299 F F F 3 060-079 173 173 173 299 299 299 F F F 4 080-099 174 174 174 300 300 300 F F F 5 100-119 158 158 158 271 271 271 F F F 6 120-139 169 169 169 292 292 292 F F F 7 140-159 166 166 166 285 285 285 F F F 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 F F F 11 220-239 175 175 175 307 307 307 F F F 12 240-259 171 171 171 296 296 296 F F F 13 260-279 172 172 172 298 298 298 F F F 14 280-299 174 174 174 303 303 303 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 171 171 171 296 296 296 F F F 17 340-359 169 169 169 292 292 292 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 174 174 174 303 303 303 F F F 21 420-439 174 174 174 305 305 305 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 175 175 175 307 307 307 F F F 25 500-519 174 174 174 306 306 306 F F F 26 520-539 174 174 174 306 306 306 F F F 27 540-559 173 173 173 299 299 299 F F F 28 560-579 174 174 174 304 304 304 F F F 29 580-599 174 174 174 304 304 304 F F F Max gen: 175, Max flush: 307 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.52743921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.539407443Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.539780127Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.539819774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.539832334Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.539850242Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"Crucible 32001286-9ec9-47b9-b1c1-7ea78a5adfeb has session id: 43693f48-4928-4e1a-8c6d-c22d8d936cfe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540187629Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Upstairs opts: Upstairs UUID: 32001286-9ec9-47b9-b1c1-7ea78a5adfeb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540224817Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Crucible stats registered with UUID: 32001286-9ec9-47b9-b1c1-7ea78a5adfeb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540240646Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:18.540253965Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540428313Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540457161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.54047206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540573573Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540591992Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"32001286-9ec9-47b9-b1c1-7ea78a5adfeb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540608421Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.54062119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540635089Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540649008Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"2","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540662727Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540676976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540691135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:33576)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540717523Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:33103)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540735402Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"ds_connection connected from Ok(127.0.0.1:60017)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.540750571Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541485041Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"2","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541514169Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541549206Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541842976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541882673Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[0]R flush_numbers[0..12]: [296, 292, 299, 299, 300, 271, 292, 285, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541945429Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[0]R generation[0..12]: [171, 169, 173, 173, 174, 158, 169, 166, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541962368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541978657Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[1]R flush_numbers[0..12]: [296, 292, 299, 299, 300, 271, 292, 285, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.541994656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[1]R generation[0..12]: [171, 169, 173, 173, 174, 158, 169, 166, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542010595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542026964Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[2]R flush_numbers[0..12]: [296, 292, 299, 299, 300, 271, 292, 285, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542045282Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[2]R generation[0..12]: [171, 169, 173, 173, 174, 158, 169, 166, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542061141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.54207723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542095189Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542109348Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542122587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542136036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"":"downstairs","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542150335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542163044Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"0","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542177403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"1","":"downstairs","session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542191292Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542208001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.54222163Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"32001286-9ec9-47b9-b1c1-7ea78a5adfeb is now active with session: 43693f48-4928-4e1a-8c6d-c22d8d936cfe","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542236699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542249238Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910,"session_id":"43693f48-4928-4e1a-8c6d-c22d8d936cfe"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542262568Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} {"msg":"Activated sub_volume 32001286-9ec9-47b9-b1c1-7ea78a5adfeb","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:18.542276307Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 12288 data: 11 15 13 02/30 Read block 553 len 28672 03/30 Flush 04/30 Write block 289 len 24576 data: 12 10 11 9 10 9 05/30 Write block 553 len 8192 data: 11 12 06/30 Flush 07/30 Write block 328 len 28672 data: 10 12 11 13 13 12 11 08/30 Flush 09/30 Write block 257 len 40960 data: 10 10 10 10 13 11 13 13 11 12 10/30 Read block 75 len 12288 11/30 Write block 420 len 16384 data: 10 12 11 12 12/30 Write block 32 len 24576 data: 15 13 13 16 16 17 13/30 Write block 143 len 24576 data: 12 11 10 8 7 8 14/30 Read block 195 len 8192 15/30 Write block 126 len 8192 data: 15 14 16/30 Write block 254 len 36864 data: 9 8 8 11 11 11 11 14 12 17/30 Read block 92 len 24576 18/30 Read block 582 len 4096 19/30 Read block 217 len 36864 20/30 Read block 197 len 20480 21/30 Read block 12 len 8192 22/30 Read block 453 len 36864 23/30 Read block 465 len 16384 24/30 Write block 36 len 12288 data: 17 18 20 25/30 Read block 292 len 16384 26/30 Write block 344 len 36864 data: 10 10 12 12 11 13 12 12 12 27/30 Write block 40 len 20480 data: 19 15 14 13 13 28/30 Write block 113 len 28672 data: 9 10 9 10 10 13 12 29/30 Read block 353 len 4096 30/30 Write block 116 len 28672 data: 11 11 14 13 13 11 11 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 6 Done Done Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 26 1 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1002)) Some(JobId(1007)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:18.55452121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1910} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 176 176 176 308 308 308 T F T <--- 2 040-059 173 173 173 299 299 299 T F T <--- 3 060-079 173 173 173 299 299 299 F F F 4 080-099 174 174 174 300 300 300 F F F 5 100-119 158 158 158 271 271 271 T F T <--- 6 120-139 169 169 169 292 292 292 T F T <--- 7 140-159 166 166 166 285 285 285 T F T <--- 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 F F F 11 220-239 175 175 175 307 307 307 F F F 12 240-259 171 171 171 296 296 296 T F T <--- 13 260-279 172 172 172 298 298 298 T F T <--- 14 280-299 176 174 176 309 303 309 F T F <--- 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 171 176 310 296 310 F F F <--- 17 340-359 169 169 169 292 292 292 T F T <--- 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 174 174 174 303 303 303 F F F 21 420-439 174 174 174 305 305 305 T F T <--- 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 175 175 175 307 307 307 F F F 25 500-519 174 174 174 306 306 306 F F F 26 520-539 174 174 174 306 306 306 F F F 27 540-559 176 173 176 309 299 309 F F F <--- 28 560-579 174 174 174 304 304 304 F F F 29 580-599 174 174 174 304 304 304 F F F Max gen: 176, Max flush: 310 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 177 177 177 311 311 311 F F F 2 040-059 177 177 177 311 311 311 F F F 3 060-079 173 173 173 299 299 299 F F F 4 080-099 174 174 174 300 300 300 F F F 5 100-119 177 177 177 311 311 311 F F F 6 120-139 177 177 177 311 311 311 F F F 7 140-159 177 177 177 311 311 311 F F F 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 F F F 11 220-239 175 175 175 307 307 307 F F F 12 240-259 177 177 177 311 311 311 F F F 13 260-279 177 177 177 311 311 311 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 177 177 177 311 311 311 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 174 174 174 303 303 303 F F F 21 420-439 177 177 177 311 311 311 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 175 175 175 307 307 307 F F F 25 500-519 174 174 174 306 306 306 F F F 26 520-539 174 174 174 306 306 306 F F F 27 540-559 176 176 176 309 309 309 F F F 28 560-579 174 174 174 304 304 304 F F F 29 580-599 174 174 174 304 304 304 F F F Max gen: 177, Max flush: 311 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.878801092Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.890548759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.890871947Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.890910834Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.890923083Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.890940802Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"Crucible c8db9e2c-ee7b-4f2a-a6e1-544bda619a49 has session id: f8698a8d-77ff-4abe-b5ad-31d5deaa3663","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891281239Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Upstairs opts: Upstairs UUID: c8db9e2c-ee7b-4f2a-a6e1-544bda619a49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891315826Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Crucible stats registered with UUID: c8db9e2c-ee7b-4f2a-a6e1-544bda619a49","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891331465Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:22.891345074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891530462Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891566079Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891581408Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891670252Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891688441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"c8db9e2c-ee7b-4f2a-a6e1-544bda619a49 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.89170385Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891716969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891730998Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891744957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891758726Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891772905Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891806223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"ds_connection connected from Ok(127.0.0.1:61212)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891822032Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"ds_connection connected from Ok(127.0.0.1:61904)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.89184398Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"ds_connection connected from Ok(127.0.0.1:54919)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.891902846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.892570961Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.892621327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.892638406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.892923677Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.892968463Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[0]R flush_numbers[0..12]: [296, 311, 311, 299, 300, 311, 311, 311, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893037169Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[0]R generation[0..12]: [171, 177, 177, 173, 174, 177, 177, 177, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893054538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893071256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[1]R flush_numbers[0..12]: [296, 311, 311, 299, 300, 311, 311, 311, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893087205Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[1]R generation[0..12]: [171, 177, 177, 173, 174, 177, 177, 177, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893102984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893119023Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[2]R flush_numbers[0..12]: [296, 311, 311, 299, 300, 311, 311, 311, 299, 298, 299, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893134622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[2]R generation[0..12]: [171, 177, 177, 173, 174, 177, 177, 177, 173, 172, 173, 175]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893150071Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.89316632Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893179839Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893198908Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893212297Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893225776Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"":"downstairs","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893241195Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893254054Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"0","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893268233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"1","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893281762Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"client":"2","":"downstairs","session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893298051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.8933119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"c8db9e2c-ee7b-4f2a-a6e1-544bda619a49 is now active with session: f8698a8d-77ff-4abe-b5ad-31d5deaa3663","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893327279Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893339768Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918,"session_id":"f8698a8d-77ff-4abe-b5ad-31d5deaa3663"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893353227Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} {"msg":"Activated sub_volume c8db9e2c-ee7b-4f2a-a6e1-544bda619a49","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:22.893364836Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 138 len 16384 data: 20 17 20 18 02/30 Write block 418 len 24576 data: 11 12 11 13 12 13 03/30 Read block 520 len 32768 04/30 Read block 530 len 4096 05/30 Read block 86 len 28672 06/30 Read block 332 len 12288 07/30 Write block 540 len 28672 data: 14 16 18 16 13 10 9 08/30 Read block 442 len 36864 09/30 Read block 320 len 28672 10/30 Flush 11/30 Flush 12/30 Write block 492 len 20480 data: 12 13 15 13 11 13/30 Read block 348 len 32768 14/30 Read block 52 len 20480 15/30 Write block 90 len 40960 data: 9 10 8 10 11 13 17 17 14 15 16/30 Read block 106 len 40960 17/30 Read block 179 len 32768 18/30 Flush 19/30 Write block 238 len 20480 data: 10 12 12 16 19 20/30 Read block 235 len 4096 21/30 Flush 22/30 Write block 265 len 8192 data: 12 13 23/30 Read block 218 len 36864 24/30 Write block 73 len 16384 data: 18 14 10 11 25/30 Write block 22 len 28672 data: 11 12 16 14 15 16 13 26/30 Read block 153 len 32768 27/30 Write block 539 len 28672 data: 13 15 17 19 17 14 11 28/30 Write block 202 len 40960 data: 17 17 17 17 18 18 17 18 16 9 29/30 Write block 402 len 40960 data: 19 16 15 14 14 16 17 17 14 15 30/30 Write block 402 len 20480 data: 20 17 16 15 15 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 24 2 28 Done 28 6 28 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:22.905977335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1918} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 177 177 177 311 311 311 T F T <--- 2 040-059 177 177 177 311 311 311 F F F 3 060-079 173 173 173 299 299 299 T F T <--- 4 080-099 178 174 178 314 300 314 F F F <--- 5 100-119 177 177 177 311 311 311 F F F 6 120-139 178 177 178 312 311 312 F T F <--- 7 140-159 178 177 178 312 311 312 F T F <--- 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 173 173 173 299 299 299 T F T <--- 11 220-239 178 175 178 315 307 315 F F F <--- 12 240-259 178 177 178 315 311 315 F F F <--- 13 260-279 177 177 177 311 311 311 T F T <--- 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 177 177 177 311 311 311 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 178 174 178 312 303 312 T T T <--- 21 420-439 178 177 178 312 311 312 F T F <--- 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 178 175 178 314 307 314 F F F <--- 25 500-519 174 174 174 306 306 306 F F F 26 520-539 174 174 174 306 306 306 T F T <--- 27 540-559 178 176 178 312 309 312 T F T <--- 28 560-579 174 174 174 304 304 304 F F F 29 580-599 174 174 174 304 304 304 F F F Max gen: 178, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 179 179 179 316 316 316 F F F 2 040-059 177 177 177 311 311 311 F F F 3 060-079 179 179 179 316 316 316 F F F 4 080-099 178 178 178 314 314 314 F F F 5 100-119 177 177 177 311 311 311 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 178 178 178 312 312 312 F F F 8 160-179 173 173 173 299 299 299 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 179 179 179 316 316 316 F F F 11 220-239 178 178 178 315 315 315 F F F 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 177 177 177 311 311 311 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 179 179 179 316 316 316 F F F 21 420-439 178 178 178 312 312 312 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 178 178 178 314 314 314 F F F 25 500-519 174 174 174 306 306 306 F F F 26 520-539 179 179 179 316 316 316 F F F 27 540-559 179 179 179 316 316 316 F F F 28 560-579 174 174 174 304 304 304 F F F 29 580-599 174 174 174 304 304 304 F F F Max gen: 179, Max flush: 316 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.235317341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.246988084Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247307612Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247354658Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247366438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247384346Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"Crucible cbd826b1-36c2-4388-acde-b9ab164d8ef1 has session id: de8009ce-9819-4f7b-9705-ef7a4dea57f7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247740272Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Upstairs opts: Upstairs UUID: cbd826b1-36c2-4388-acde-b9ab164d8ef1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.2477709Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Crucible stats registered with UUID: cbd826b1-36c2-4388-acde-b9ab164d8ef1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.247786229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:27.247800098Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248017793Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248042721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.24805713Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248196241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.24821331Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"cbd826b1-36c2-4388-acde-b9ab164d8ef1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248245678Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248259217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248273376Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248287365Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248300754Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248314933Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248331132Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35826)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248346551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50830)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.24836146Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49921)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.248377719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.24909394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249145786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249245539Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249442976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249479723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[0]R flush_numbers[0..12]: [296, 316, 311, 316, 314, 311, 312, 312, 299, 298, 316, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249617844Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[0]R generation[0..12]: [171, 179, 177, 179, 178, 177, 178, 178, 173, 172, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249635483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249652161Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[1]R flush_numbers[0..12]: [296, 316, 311, 316, 314, 311, 312, 312, 299, 298, 316, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.2496681Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[1]R generation[0..12]: [171, 179, 177, 179, 178, 177, 178, 178, 173, 172, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249684479Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249705398Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[2]R flush_numbers[0..12]: [296, 316, 311, 316, 314, 311, 312, 312, 299, 298, 316, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249721877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[2]R generation[0..12]: [171, 179, 177, 179, 178, 177, 178, 178, 173, 172, 179, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249737896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249754255Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249767744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","Wait for a query_work_queue command to finish before sending IO pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249787952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249801431Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.24981477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"":"downstairs","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249828919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249841529Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"0","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249855388Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"1","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249869287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"client":"2","":"downstairs","session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249883326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-16T21:38:27.249896495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"cbd826b1-36c2-4388-acde-b9ab164d8ef1 is now active with session: de8009ce-9819-4f7b-9705-ef7a4dea57f7","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249913684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249928293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926,"session_id":"de8009ce-9819-4f7b-9705-ef7a4dea57f7"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249942062Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} {"msg":"Activated sub_volume cbd826b1-36c2-4388-acde-b9ab164d8ef1","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:27.249953521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 557 len 16384 data: 12 15 12 11 03/30 Write block 425 len 8192 data: 14 15 04/30 Write block 547 len 28672 data: 10 11 11 9 8 9 12 05/30 Write block 347 len 36864 data: 13 12 14 13 13 13 13 15 19 06/30 Flush 07/30 Write block 433 len 12288 data: 14 14 15 08/30 Read block 46 len 24576 09/30 Read block 349 len 8192 10/30 Read block 467 len 12288 11/30 Read block 463 len 32768 12/30 Write block 80 len 16384 data: 12 15 11 8 13/30 Write block 165 len 16384 data: 14 12 11 12 14/30 Read block 363 len 24576 15/30 Write block 545 len 32768 data: 12 10 11 12 12 10 9 10 16/30 Flush 17/30 Write block 106 len 40960 data: 10 7 7 6 8 9 10 10 11 10 18/30 Write block 493 len 24576 data: 14 16 14 12 10 15 19/30 Read block 437 len 40960 20/30 Read block 513 len 36864 21/30 Write block 496 len 16384 data: 13 11 16 16 22/30 Flush 23/30 Write block 523 len 20480 data: 12 10 9 11 14 24/30 Write block 220 len 20480 data: 19 15 17 19 20 25/30 Read block 10 len 8192 26/30 Write block 586 len 16384 data: 12 10 7 9 27/30 Write block 20 len 28672 data: 13 15 12 13 17 15 16 28/30 Read block 10 len 8192 29/30 Write block 500 len 32768 data: 16 18 21 24 22 22 20 21 30/30 Write block 423 len 28672 data: 14 14 15 16 15 17 19 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:27.261517141Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1926} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 179 179 179 316 316 316 T T F <--- 2 040-059 177 177 177 311 311 311 F F F 3 060-079 179 179 179 316 316 316 F F F 4 080-099 180 180 178 319 319 314 F F F <--- 5 100-119 180 180 177 320 320 311 F F F <--- 6 120-139 178 178 178 312 312 312 F F F 7 140-159 178 178 178 312 312 312 F F F 8 160-179 180 180 173 319 319 299 F F F <--- 9 180-199 172 172 172 298 298 298 F F F 10 200-219 179 179 179 316 316 316 F F F 11 220-239 178 178 178 315 315 315 T T F <--- 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 180 180 177 318 318 311 F F F <--- 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 179 179 179 316 316 316 F F F 21 420-439 180 180 178 319 319 312 T T F <--- 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 180 180 178 320 320 314 F F F <--- 25 500-519 174 174 174 306 306 306 T T F <--- 26 520-539 179 179 179 316 316 316 T T F <--- 27 540-559 180 180 179 319 319 316 F F F <--- 28 560-579 180 180 174 318 318 304 F F F <--- 29 580-599 174 174 174 304 304 304 T T F <--- Max gen: 180, Max flush: 320 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 177 177 177 311 311 311 F F F 3 060-079 179 179 179 316 316 316 F F F 4 080-099 180 180 180 319 319 319 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 178 178 178 312 312 312 F F F 8 160-179 180 180 180 319 319 319 F F F 9 180-199 172 172 172 298 298 298 F F F 10 200-219 179 179 179 316 316 316 F F F 11 220-239 181 181 181 321 321 321 F F F 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 180 180 180 318 318 318 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 179 179 179 316 316 316 F F F 21 420-439 181 181 181 321 321 321 F F F 22 440-459 167 167 167 289 289 289 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 181 181 181 321 321 321 F F F 26 520-539 181 181 181 321 321 321 F F F 27 540-559 180 180 180 319 319 319 F F F 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 181, Max flush: 321 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.584846778Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.596562977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.596888825Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.596931082Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.596943491Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.59696134Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"Crucible 7cb84b2e-5035-42f0-a061-85afaefb2eb5 has session id: 33765fca-f249-4002-baaf-21b5f886407e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597272929Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Upstairs opts: Upstairs UUID: 7cb84b2e-5035-42f0-a061-85afaefb2eb5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597307766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Crucible stats registered with UUID: 7cb84b2e-5035-42f0-a061-85afaefb2eb5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597323895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:31.597337554Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597502043Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597527561Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.59754562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597648003Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597667642Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"7cb84b2e-5035-42f0-a061-85afaefb2eb5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.59769843Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597712209Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597726118Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597740037Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597753416Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597767735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597781684Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34838)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597830441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61606)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597846829Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41531)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.597861858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.598560121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.598594008Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.598674403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.598937495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.598985882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[0]R flush_numbers[0..12]: [296, 321, 311, 316, 319, 320, 312, 312, 319, 298, 316, 321]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599050637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[0]R generation[0..12]: [171, 181, 177, 179, 180, 180, 178, 178, 180, 172, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599068016Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599084595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[1]R flush_numbers[0..12]: [296, 321, 311, 316, 319, 320, 312, 312, 319, 298, 316, 321]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599100794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[1]R generation[0..12]: [171, 181, 177, 179, 180, 180, 178, 178, 180, 172, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599116583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599133072Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[2]R flush_numbers[0..12]: [296, 321, 311, 316, 319, 320, 312, 312, 319, 298, 316, 321]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.59914903Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[2]R generation[0..12]: [171, 181, 177, 179, 180, 180, 178, 178, 180, 172, 179, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599164879Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599181468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599199977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599213926Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599229235Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599242744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"":"downstairs","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599256853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599269342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"0","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599283031Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"1","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.59929967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"client":"2","":"downstairs","session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599314459Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599327918Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"7cb84b2e-5035-42f0-a061-85afaefb2eb5 is now active with session: 33765fca-f249-4002-baaf-21b5f886407e","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599342907Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599355296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934,"session_id":"33765fca-f249-4002-baaf-21b5f886407e"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599368366Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} {"msg":"Activated sub_volume 7cb84b2e-5035-42f0-a061-85afaefb2eb5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:31.599380045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 148 len 4096 data: 9 02/30 Read block 341 len 12288 03/30 Read block 358 len 16384 04/30 Read block 277 len 28672 05/30 Write block 228 len 20480 data: 20 18 21 19 20 06/30 Write block 448 len 8192 data: 13 9 07/30 Flush 08/30 Write block 195 len 32768 data: 9 10 12 13 13 14 15 18 09/30 Read block 395 len 24576 10/30 Write block 410 len 40960 data: 15 16 15 13 13 12 13 13 12 13 11/30 Read block 175 len 4096 12/30 Read block 337 len 28672 13/30 Read block 441 len 4096 14/30 Write block 158 len 8192 data: 7 5 15/30 Read block 340 len 28672 16/30 Write block 438 len 32768 data: 15 10 9 12 12 13 10 12 17/30 Read block 409 len 28672 18/30 Read block 26 len 40960 19/30 Flush 20/30 Write block 404 len 16384 data: 17 16 16 17 21/30 Read block 208 len 4096 22/30 Write block 155 len 24576 data: 8 7 8 8 6 9 23/30 Flush 24/30 Write block 95 len 20480 data: 14 18 18 15 16 25/30 Read block 339 len 4096 26/30 Write block 547 len 36864 data: 12 13 13 11 10 11 13 13 13 27/30 Read block 550 len 28672 28/30 Read block 525 len 36864 29/30 Read block 131 len 28672 30/30 Read block 85 len 32768 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 27 28 Done 29 30 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:31.612302582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1934} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 177 177 177 311 311 311 F F F 3 060-079 179 179 179 316 316 316 F F F 4 080-099 180 180 180 319 319 319 T T F <--- 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 178 324 324 312 F F T <--- 8 160-179 182 182 180 324 324 319 F F F <--- 9 180-199 182 182 172 323 323 298 F F F <--- 10 200-219 182 182 179 323 323 316 F F F <--- 11 220-239 182 182 181 322 322 321 F F F <--- 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 180 180 180 318 318 318 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 182 182 179 324 324 316 F F F <--- 21 420-439 182 182 181 323 323 321 F F F <--- 22 440-459 182 182 167 323 323 289 F F F <--- 23 460-479 174 174 174 304 304 304 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 181 181 181 321 321 321 F F F 26 520-539 181 181 181 321 321 321 F F F 27 540-559 180 180 180 319 319 319 T T F <--- 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 182, Max flush: 324 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 177 177 177 311 311 311 F F F 3 060-079 179 179 179 316 316 316 F F F 4 080-099 183 183 183 325 325 325 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 182 324 324 324 F F F 8 160-179 182 182 182 324 324 324 F F F 9 180-199 182 182 182 323 323 323 F F F 10 200-219 182 182 182 323 323 323 F F F 11 220-239 182 182 182 322 322 322 F F F 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 176 176 176 309 309 309 F F F 15 300-319 171 171 171 296 296 296 F F F 16 320-339 176 176 176 310 310 310 F F F 17 340-359 180 180 180 318 318 318 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 175 175 175 307 307 307 F F F 20 400-419 182 182 182 324 324 324 F F F 21 420-439 182 182 182 323 323 323 F F F 22 440-459 182 182 182 323 323 323 F F F 23 460-479 174 174 174 304 304 304 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 181 181 181 321 321 321 F F F 26 520-539 181 181 181 321 321 321 F F F 27 540-559 183 183 183 325 325 325 F F F 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 183, Max flush: 325 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.929288952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.940932986Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941255504Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941296311Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.94131232Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941330609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"Crucible bca36cb8-aa7f-4bc9-9814-8be693a991fa has session id: 67573326-e6c2-432c-91a2-a6e0eaf7f36d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941654817Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Upstairs opts: Upstairs UUID: bca36cb8-aa7f-4bc9-9814-8be693a991fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941702004Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Crucible stats registered with UUID: bca36cb8-aa7f-4bc9-9814-8be693a991fa","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941717832Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:35.941730982Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941917309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941952906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.941968315Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942079998Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942097147Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"bca36cb8-aa7f-4bc9-9814-8be693a991fa active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942112026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942124525Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942138264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"1","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942152053Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942165712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942183551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.94219796Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57213)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942299303Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58922)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.942326901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46257)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.94234274Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943054241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"1","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943082199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943159644Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943440135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943465143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[0]R flush_numbers[0..12]: [296, 321, 311, 316, 325, 320, 312, 324, 324, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943502491Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[0]R generation[0..12]: [171, 181, 177, 179, 183, 180, 178, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943520699Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943537508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[1]R flush_numbers[0..12]: [296, 321, 311, 316, 325, 320, 312, 324, 324, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943553587Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[1]R generation[0..12]: [171, 181, 177, 179, 183, 180, 178, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943572446Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943588885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[2]R flush_numbers[0..12]: [296, 321, 311, 316, 325, 320, 312, 324, 324, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943604734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[2]R generation[0..12]: [171, 181, 177, 179, 183, 180, 178, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943620362Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2026-03-16T21:38:35.943636721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.94365597Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943670229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943683708Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943697007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"":"downstairs","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943710976Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943723895Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"0","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943738434Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943754773Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"client":"2","":"downstairs","session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943769542Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943783021Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"bca36cb8-aa7f-4bc9-9814-8be693a991fa is now active with session: 67573326-e6c2-432c-91a2-a6e0eaf7f36d","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.94379962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943812509Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942,"session_id":"67573326-e6c2-432c-91a2-a6e0eaf7f36d"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943825588Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} {"msg":"Activated sub_volume bca36cb8-aa7f-4bc9-9814-8be693a991fa","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:35.943836948Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 24576 data: 17 15 15 15 12 11 02/30 Read block 58 len 8192 03/30 Read block 11 len 28672 04/30 Write block 49 len 28672 data: 14 13 12 13 9 8 9 05/30 Write block 511 len 40960 data: 13 13 10 11 14 16 19 20 17 17 06/30 Read block 158 len 4096 07/30 Read block 185 len 8192 08/30 Read block 105 len 8192 09/30 Write block 385 len 8192 data: 15 17 10/30 Write block 316 len 8192 data: 20 22 11/30 Write block 198 len 12288 data: 14 14 15 12/30 Read block 477 len 24576 13/30 Write block 291 len 16384 data: 12 10 11 10 14/30 Write block 77 len 4096 data: 10 15/30 Read block 411 len 24576 16/30 Write block 176 len 24576 data: 12 12 13 16 18 19 17/30 Write block 392 len 40960 data: 13 18 18 17 22 22 23 20 22 17 18/30 Write block 463 len 20480 data: 12 12 12 15 16 19/30 Read block 117 len 8192 20/30 Flush 21/30 Write block 410 len 4096 data: 16 22/30 Read block 140 len 24576 23/30 Read block 54 len 12288 24/30 Write block 97 len 20480 data: 19 16 17 15 14 25/30 Read block 339 len 24576 26/30 Flush 27/30 Write block 49 len 28672 data: 15 14 13 14 10 9 10 28/30 Read block 151 len 16384 29/30 Write block 454 len 36864 data: 16 13 19 20 17 18 19 14 13 30/30 Write block 455 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 2 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:35.955179393Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1942} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 184 184 177 326 326 311 T T F <--- 3 060-079 184 184 179 326 326 316 F F F <--- 4 080-099 184 184 183 327 327 325 F F F <--- 5 100-119 184 184 180 327 327 320 F F F <--- 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 182 324 324 324 F F F 8 160-179 184 184 182 326 326 324 F F F <--- 9 180-199 184 184 182 326 326 323 F F F <--- 10 200-219 184 184 182 326 326 323 F F F <--- 11 220-239 182 182 182 322 322 322 F F F 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 184 184 176 326 326 309 F F F <--- 15 300-319 184 184 171 326 326 296 F F T <--- 16 320-339 184 184 176 326 326 310 F F T <--- 17 340-359 180 180 180 318 318 318 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 184 184 175 326 326 307 F F F <--- 20 400-419 184 184 182 327 327 324 F F F <--- 21 420-439 182 182 182 323 323 323 F F F 22 440-459 182 182 182 323 323 323 T T F <--- 23 460-479 184 184 174 326 326 304 T T F <--- 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 181 326 326 321 F F F <--- 26 520-539 184 184 181 326 326 321 F F F <--- 27 540-559 183 183 183 325 325 325 F F F 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 184, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 185 185 185 328 328 328 F F F 3 060-079 184 184 184 326 326 326 F F F 4 080-099 184 184 184 327 327 327 F F F 5 100-119 184 184 184 327 327 327 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 182 324 324 324 F F F 8 160-179 184 184 184 326 326 326 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 184 184 184 326 326 326 F F F 11 220-239 182 182 182 322 322 322 F F F 12 240-259 178 178 178 315 315 315 F F F 13 260-279 179 179 179 316 316 316 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 184 184 184 326 326 326 F F F 17 340-359 180 180 180 318 318 318 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 184 184 184 326 326 326 F F F 20 400-419 184 184 184 327 327 327 F F F 21 420-439 182 182 182 323 323 323 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 184 184 184 326 326 326 F F F 27 540-559 183 183 183 325 325 325 F F F 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 185, Max flush: 328 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.28567354Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.297340803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.29767262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.297715327Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.297728406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.297746475Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"Crucible 196290af-99c9-479b-894e-c04e7d7d20a0 has session id: e7a57798-4036-484d-803f-f7fd235bd94b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298040485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Upstairs opts: Upstairs UUID: 196290af-99c9-479b-894e-c04e7d7d20a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298074863Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Crucible stats registered with UUID: 196290af-99c9-479b-894e-c04e7d7d20a0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298090262Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:40.298103521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298302997Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298339575Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298355074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298462326Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298479335Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"196290af-99c9-479b-894e-c04e7d7d20a0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298511183Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298525182Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298539311Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.29855297Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298567099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298601287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298624015Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38498)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298639704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38530)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298770215Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46079)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.298789384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299382693Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299412811Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.29942938Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299765877Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299781126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[0]R flush_numbers[0..12]: [296, 321, 328, 326, 327, 327, 312, 324, 326, 326, 326, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299818484Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[0]R generation[0..12]: [171, 181, 185, 184, 184, 184, 178, 182, 184, 184, 184, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299835622Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299852391Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[1]R flush_numbers[0..12]: [296, 321, 328, 326, 327, 327, 312, 324, 326, 326, 326, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.29986797Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[1]R generation[0..12]: [171, 181, 185, 184, 184, 184, 178, 182, 184, 184, 184, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299883569Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299899718Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[2]R flush_numbers[0..12]: [296, 321, 328, 326, 327, 327, 312, 324, 326, 326, 326, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299915647Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[2]R generation[0..12]: [171, 181, 185, 184, 184, 184, 178, 182, 184, 184, 184, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299931216Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299947265Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299960734Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299976913Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.299990672Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300004301Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"":"downstairs","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.30002387Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300037129Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"0","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300051088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"1","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300064927Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"client":"2","":"downstairs","session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300079016Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300092075Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"196290af-99c9-479b-894e-c04e7d7d20a0 is now active with session: e7a57798-4036-484d-803f-f7fd235bd94b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300106824Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300122223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950,"session_id":"e7a57798-4036-484d-803f-f7fd235bd94b"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300135742Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} {"msg":"Activated sub_volume 196290af-99c9-479b-894e-c04e7d7d20a0","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:40.300147261Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 12288 data: 16 15 16 02/30 Read block 147 len 32768 03/30 Flush 04/30 Write block 44 len 16384 data: 14 13 14 12 05/30 Read block 484 len 28672 06/30 Read block 589 len 12288 07/30 Read block 282 len 32768 08/30 Write block 559 len 4096 data: 13 09/30 Read block 281 len 16384 10/30 Flush 11/30 Flush 12/30 Write block 523 len 40960 data: 13 11 10 12 15 16 16 16 13 12 13/30 Read block 283 len 20480 14/30 Read block 336 len 4096 15/30 Read block 281 len 16384 16/30 Write block 224 len 40960 data: 21 19 18 22 21 19 22 20 21 20 17/30 Read block 146 len 8192 18/30 Write block 329 len 40960 data: 13 12 14 14 13 12 16 16 15 15 19/30 Read block 414 len 32768 20/30 Read block 116 len 36864 21/30 Flush 22/30 Write block 257 len 32768 data: 12 12 12 12 15 13 14 14 23/30 Write block 347 len 24576 data: 14 13 15 14 14 14 24/30 Write block 404 len 28672 data: 18 17 17 18 18 18 17 25/30 Flush 26/30 Write block 209 len 8192 data: 19 17 27/30 Read block 188 len 12288 28/30 Write block 60 len 36864 data: 13 13 14 18 20 18 18 19 21 29/30 Write block 112 len 20480 data: 11 11 12 11 12 30/30 Read block 474 len 8192 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:40.313101956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1950} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 186 186 185 330 330 328 F F F <--- 3 060-079 184 184 184 326 326 326 T T F <--- 4 080-099 184 184 184 327 327 327 F F F 5 100-119 184 184 184 327 327 327 T T F <--- 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 182 324 324 324 F F F 8 160-179 184 184 184 326 326 326 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 184 184 184 326 326 326 T T F <--- 11 220-239 186 186 182 332 332 322 F F F <--- 12 240-259 186 186 178 333 333 315 F F F <--- 13 260-279 186 186 179 333 333 316 F F F <--- 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 186 186 184 332 332 326 F F F <--- 17 340-359 186 186 180 333 333 318 F F F <--- 18 360-379 175 175 175 307 307 307 F F F 19 380-399 186 186 186 329 329 329 F F F 20 400-419 186 186 184 333 333 327 F F F <--- 21 420-439 182 182 182 323 323 323 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 186 186 184 332 332 326 F F F <--- 27 540-559 186 186 183 330 330 325 F F F <--- 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 186, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 181 181 181 321 321 321 F F F 2 040-059 186 186 186 330 330 330 F F F 3 060-079 187 187 187 334 334 334 F F F 4 080-099 184 184 184 327 327 327 F F F 5 100-119 187 187 187 334 334 334 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 182 182 182 324 324 324 F F F 8 160-179 184 184 184 326 326 326 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 186 186 332 332 332 F F F 12 240-259 186 186 186 333 333 333 F F F 13 260-279 186 186 186 333 333 333 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 186 186 186 332 332 332 F F F 17 340-359 186 186 186 333 333 333 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 186 186 186 329 329 329 F F F 20 400-419 186 186 186 333 333 333 F F F 21 420-439 182 182 182 323 323 323 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 186 186 186 330 330 330 F F F 28 560-579 180 180 180 318 318 318 F F F 29 580-599 181 181 181 321 321 321 F F F Max gen: 187, Max flush: 334 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.634985972Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.646634406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.646959144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.646995721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.64701158Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647029839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"Crucible 9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b has session id: d2c69159-ad50-4caa-baff-63608368eccc","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647368406Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Upstairs opts: Upstairs UUID: 9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647415242Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Crucible stats registered with UUID: 9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647431101Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:44.64744452Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647618329Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647654156Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647670135Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647771868Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647789047Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647819735Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647833984Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647848553Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"1","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647905739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647931287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647946906Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647961405Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"ds_connection connected from Ok(127.0.0.1:56441)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647976294Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63320)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.647991403Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51799)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.648006582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.648721153Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.648760971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"1","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.648778339Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649096958Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649141665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[0]R flush_numbers[0..12]: [296, 321, 330, 334, 327, 334, 312, 324, 326, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.64921187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[0]R generation[0..12]: [171, 181, 186, 187, 184, 187, 178, 182, 184, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649229199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649245977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[1]R flush_numbers[0..12]: [296, 321, 330, 334, 327, 334, 312, 324, 326, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649264766Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[1]R generation[0..12]: [171, 181, 186, 187, 184, 187, 178, 182, 184, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649280695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649297034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[2]R flush_numbers[0..12]: [296, 321, 330, 334, 327, 334, 312, 324, 326, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649312993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[2]R generation[0..12]: [171, 181, 186, 187, 184, 187, 178, 182, 184, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649328582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649344701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"Wait for a query_work_queue command to finish before sending IO msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649363979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649378598Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649392067Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649404977Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"":"downstairs","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649419126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649431785Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"0","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649446024Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649462013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"client":"2","":"downstairs","session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649478372Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649491941Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b is now active with session: d2c69159-ad50-4caa-baff-63608368eccc","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.64950719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649519839Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958,"session_id":"d2c69159-ad50-4caa-baff-63608368eccc"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649533148Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} {"msg":"Activated sub_volume 9d9b51c1-d9b4-4460-ba20-cdebaa9f3c4b","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:44.649544647Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 36864 data: 10 9 12 13 14 14 12 13 11 02/30 Read block 489 len 40960 03/30 Write block 171 len 24576 data: 12 14 13 10 11 13 04/30 Write block 151 len 40960 data: 11 10 9 7 9 8 9 9 7 10 05/30 Read block 88 len 32768 06/30 Read block 155 len 40960 07/30 Write block 592 len 16384 data: 9 9 7 7 08/30 Read block 566 len 28672 09/30 Read block 101 len 12288 10/30 Read block 96 len 16384 11/30 Read block 145 len 32768 12/30 Write block 152 len 36864 data: 11 10 8 10 9 10 10 8 11 13/30 Read block 80 len 4096 14/30 Write block 36 len 24576 data: 18 19 21 20 20 16 15/30 Read block 55 len 28672 16/30 Read block 183 len 16384 17/30 Read block 564 len 40960 18/30 Read block 532 len 28672 19/30 Read block 192 len 12288 20/30 Read block 597 len 8192 21/30 Write block 247 len 40960 data: 12 14 12 11 10 7 8 10 9 9 22/30 Read block 464 len 36864 23/30 Read block 309 len 20480 24/30 Write block 541 len 8192 data: 18 20 25/30 Read block 189 len 12288 26/30 Flush 27/30 Write block 339 len 20480 data: 13 11 13 13 11 28/30 Read block 230 len 8192 29/30 Read block 230 len 4096 30/30 Read block 509 len 20480 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 25 26 Done 30 29 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:44.662845438Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1958} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 188 188 181 335 335 321 F F F <--- 2 040-059 188 188 186 335 335 330 F F F <--- 3 060-079 187 187 187 334 334 334 F F F 4 080-099 184 184 184 327 327 327 F F F 5 100-119 187 187 187 334 334 334 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 188 188 182 335 335 324 F F T <--- 8 160-179 188 188 184 335 335 326 F F T <--- 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 186 186 332 332 332 F F F 12 240-259 188 188 186 335 335 333 F F F <--- 13 260-279 186 186 186 333 333 333 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 186 186 186 332 332 332 T T F <--- 17 340-359 186 186 186 333 333 333 T T F <--- 18 360-379 175 175 175 307 307 307 F F F 19 380-399 186 186 186 329 329 329 F F F 20 400-419 186 186 186 333 333 333 F F F 21 420-439 182 182 182 323 323 323 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 188 188 186 335 335 330 F F F <--- 28 560-579 188 188 180 335 335 318 F F T <--- 29 580-599 188 188 181 335 335 321 F F T <--- Max gen: 188, Max flush: 335 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 296 296 296 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 188 188 188 335 335 335 F F F 3 060-079 187 187 187 334 334 334 F F F 4 080-099 184 184 184 327 327 327 F F F 5 100-119 187 187 187 334 334 334 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 188 188 188 335 335 335 F F F 8 160-179 188 188 188 335 335 335 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 186 186 332 332 332 F F F 12 240-259 188 188 188 335 335 335 F F F 13 260-279 186 186 186 333 333 333 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 189 189 189 336 336 336 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 175 175 175 307 307 307 F F F 19 380-399 186 186 186 329 329 329 F F F 20 400-419 186 186 186 333 333 333 F F F 21 420-439 182 182 182 323 323 323 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 186 186 186 332 332 332 F F F 27 540-559 188 188 188 335 335 335 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 189, Max flush: 336 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.979497251Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.991146245Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.991467723Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.9915057Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99151852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.991536368Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"Crucible 0205acaa-be49-46e4-a145-b951836c3416 has session id: 1fae94d7-c608-456d-897c-5e20537a99d8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.991869416Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Upstairs opts: Upstairs UUID: 0205acaa-be49-46e4-a145-b951836c3416, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99194852Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Crucible stats registered with UUID: 0205acaa-be49-46e4-a145-b951836c3416","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.991965299Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:48.991992897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992143887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992180744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992196063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992308026Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992328114Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"0205acaa-be49-46e4-a145-b951836c3416 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992345743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992358712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992372691Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99238659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992437447Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992461755Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992476744Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38049)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992492233Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42142)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992557229Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65388)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.992584157Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99326595Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993304468Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993321786Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993652264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99370144Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[0]R flush_numbers[0..12]: [296, 335, 335, 334, 327, 334, 312, 335, 335, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993738018Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[0]R generation[0..12]: [171, 188, 188, 187, 184, 187, 178, 188, 188, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993755197Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993771536Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[1]R flush_numbers[0..12]: [296, 335, 335, 334, 327, 334, 312, 335, 335, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993787605Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[1]R generation[0..12]: [171, 188, 188, 187, 184, 187, 178, 188, 188, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993803293Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993819512Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[2]R flush_numbers[0..12]: [296, 335, 335, 334, 327, 334, 312, 335, 335, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993835441Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[2]R generation[0..12]: [171, 188, 188, 187, 184, 187, 178, 188, 188, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99385048Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993866869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993882948Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993903167Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993916576Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993930345Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"":"downstairs","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993944384Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993957223Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"0","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993971112Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"1","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.993985001Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"client":"2","":"downstairs","session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"All required reconciliation work is completedDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.99400231Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.994015719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"0205acaa-be49-46e4-a145-b951836c3416 is now active with session: 1fae94d7-c608-456d-897c-5e20537a99d8","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.994030838Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.994043717Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966,"session_id":"1fae94d7-c608-456d-897c-5e20537a99d8"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.994057236Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} {"msg":"Activated sub_volume 0205acaa-be49-46e4-a145-b951836c3416","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:48.994068935Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 12288 data: 12 13 15 02/30 Write block 155 len 20480 data: 11 10 11 11 9 03/30 Read block 239 len 24576 04/30 Write block 548 len 8192 data: 14 14 05/30 Read block 107 len 28672 06/30 Read block 150 len 32768 07/30 Write block 481 len 12288 data: 18 17 15 08/30 Read block 151 len 28672 09/30 Write block 274 len 16384 data: 12 12 9 11 10/30 Read block 113 len 8192 11/30 Flush 12/30 Write block 82 len 20480 data: 12 9 8 7 6 13/30 Read block 49 len 8192 14/30 Write block 164 len 8192 data: 12 15 15/30 Read block 215 len 40960 16/30 Write block 532 len 16384 data: 13 14 15 11 17/30 Write block 10 len 4096 data: 17 18/30 Write block 112 len 32768 data: 12 12 13 12 13 12 15 14 19/30 Read block 269 len 16384 20/30 Read block 1 len 36864 21/30 Read block 351 len 36864 22/30 Read block 271 len 20480 23/30 Flush 24/30 Write block 249 len 32768 data: 13 12 11 8 9 11 10 10 25/30 Write block 399 len 28672 data: 21 23 18 21 18 19 18 26/30 Flush 27/30 Write block 334 len 8192 data: 13 17 28/30 Write block 370 len 4096 data: 15 29/30 Write block 424 len 28672 data: 15 16 17 16 18 20 18 30/30 Write block 396 len 24576 data: 23 23 24 22 24 19 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:49.005201715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1966} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 171 338 338 296 F F F <--- 1 020-039 188 188 188 335 335 335 F F F 2 040-059 188 188 188 335 335 335 F F F 3 060-079 187 187 187 334 334 334 F F F 4 080-099 190 190 184 338 338 327 F F F <--- 5 100-119 190 190 187 338 338 334 F F F <--- 6 120-139 178 178 178 312 312 312 F F F 7 140-159 190 190 188 337 337 335 F F F <--- 8 160-179 190 190 188 338 338 335 F F F <--- 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 186 186 332 332 332 F F F 12 240-259 190 190 188 339 339 335 F F F <--- 13 260-279 190 190 186 337 337 333 F F F <--- 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 189 189 189 336 336 336 T T F <--- 17 340-359 189 189 189 336 336 336 F F F 18 360-379 175 175 175 307 307 307 T T F <--- 19 380-399 190 190 186 339 339 329 T T F <--- 20 400-419 190 190 186 339 339 333 T T F <--- 21 420-439 182 182 182 323 323 323 T T F <--- 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 190 190 180 337 337 320 F F F <--- 25 500-519 184 184 184 326 326 326 F F F 26 520-539 190 190 186 338 338 332 F F F <--- 27 540-559 190 190 188 337 337 335 F F F <--- 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 190, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 188 188 188 335 335 335 F F F 3 060-079 187 187 187 334 334 334 F F F 4 080-099 190 190 190 338 338 338 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 178 178 178 312 312 312 F F F 7 140-159 190 190 190 337 337 337 F F F 8 160-179 190 190 190 338 338 338 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 186 186 332 332 332 F F F 12 240-259 190 190 190 339 339 339 F F F 13 260-279 190 190 190 337 337 337 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 191 191 191 340 340 340 F F F 17 340-359 189 189 189 336 336 336 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 191 191 191 340 340 340 F F F 20 400-419 191 191 191 340 340 340 F F F 21 420-439 191 191 191 340 340 340 F F F 22 440-459 185 185 185 328 328 328 F F F 23 460-479 185 185 185 328 328 328 F F F 24 480-499 190 190 190 337 337 337 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 190 190 190 338 338 338 F F F 27 540-559 190 190 190 337 337 337 F F F 28 560-579 188 188 188 335 335 335 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 191, Max flush: 340 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.335161739Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:53.345849638Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.345904485Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.356327332Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.356626322Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.356663869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.356676069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.356693697Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"Crucible e6a70485-5b0e-4795-a2ea-d71128546ea9 has session id: 05fa5312-f4ce-4e5c-ba07-61b4377912df","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357055743Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Upstairs opts: Upstairs UUID: e6a70485-5b0e-4795-a2ea-d71128546ea9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357101759Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Crucible stats registered with UUID: e6a70485-5b0e-4795-a2ea-d71128546ea9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357117808Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:53.357131277Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357316815Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357352772Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357368211Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357477014Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357494093Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"e6a70485-5b0e-4795-a2ea-d71128546ea9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357511461Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357524521Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.35753876Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357553099Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357566858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357581237Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357595626Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"ds_connection connected from Ok(127.0.0.1:61970)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357610715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"ds_connection connected from Ok(127.0.0.1:42778)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357625774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"ds_connection connected from Ok(127.0.0.1:56645)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.357640483Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358398971Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358427269Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358446698Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358761846Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358804803Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[0]R flush_numbers[0..12]: [338, 335, 335, 334, 338, 338, 312, 337, 338, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.35884809Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[0]R generation[0..12]: [190, 188, 188, 187, 190, 190, 178, 190, 190, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358865659Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358882128Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[1]R flush_numbers[0..12]: [338, 335, 335, 334, 338, 338, 312, 337, 338, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358898187Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[1]R generation[0..12]: [190, 188, 188, 187, 190, 190, 178, 190, 190, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358913656Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358929885Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[2]R flush_numbers[0..12]: [338, 335, 335, 334, 338, 338, 312, 337, 338, 326, 334, 332]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358945853Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[2]R generation[0..12]: [190, 188, 188, 187, 190, 190, 178, 190, 190, 184, 187, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358961492Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.358988701Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359008889Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359023298Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359036837Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359050696Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"":"downstairs","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359064865Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359079794Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"0","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359093953Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"1","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359110842Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"client":"2","":"downstairs","session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359126081Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.35913988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"e6a70485-5b0e-4795-a2ea-d71128546ea9 is now active with session: 05fa5312-f4ce-4e5c-ba07-61b4377912df","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359155119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359167988Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974,"session_id":"05fa5312-f4ce-4e5c-ba07-61b4377912df"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359181397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} {"msg":"Activated sub_volume e6a70485-5b0e-4795-a2ea-d71128546ea9","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:53.359193437Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 442 len 4096 data: 13 02/30 Flush 03/30 Write block 539 len 36864 data: 14 16 19 21 18 15 13 11 13 04/30 Write block 392 len 36864 data: 14 19 19 18 24 24 25 23 25 05/30 Read block 361 len 16384 06/30 Read block 88 len 24576 07/30 Flush 08/30 Flush 09/30 Write block 418 len 24576 data: 13 14 12 14 13 15 10/30 Read block 370 len 32768 11/30 Write block 66 len 24576 data: 19 20 22 20 17 18 12/30 Write block 41 len 20480 data: 17 15 14 15 14 13/30 Read block 255 len 24576 14/30 Read block 220 len 28672 15/30 Write block 379 len 8192 data: 17 18 16/30 Read block 192 len 8192 17/30 Read block 90 len 20480 18/30 Write block 562 len 16384 data: 13 13 15 12 19/30 Write block 233 len 28672 data: 21 21 17 16 12 11 13 20/30 Read block 284 len 20480 21/30 Write block 167 len 8192 data: 12 13 22/30 Read block 25 len 12288 23/30 Flush 24/30 Write block 338 len 24576 data: 16 14 12 14 14 12 25/30 Write block 258 len 12288 data: 13 13 13 26/30 Read block 163 len 36864 27/30 Write block 128 len 32768 data: 13 14 13 13 14 13 13 13 28/30 Write block 463 len 12288 data: 13 13 13 29/30 Write block 565 len 36864 data: 13 14 12 11 8 12 12 9 7 30/30 Read block 178 len 4096 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 9 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 2 30 Done 0 28 26 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:53.371837383Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1974} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 188 192 192 335 344 344 F F F <--- 3 060-079 187 192 192 334 344 344 F F F <--- 4 080-099 190 190 190 338 338 338 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 178 178 178 312 312 312 F T T <--- 7 140-159 190 190 190 337 337 337 F F F 8 160-179 190 192 192 338 344 344 F F F <--- 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 186 192 192 332 344 344 F F F <--- 12 240-259 190 190 190 339 339 339 F T T <--- 13 260-279 190 190 190 337 337 337 F T T <--- 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 191 191 191 340 340 340 F T T <--- 17 340-359 189 189 189 336 336 336 F T T <--- 18 360-379 191 192 192 340 344 344 F F F <--- 19 380-399 191 192 192 340 344 344 F F F <--- 20 400-419 191 192 192 340 344 344 F F F <--- 21 420-439 191 192 192 340 344 344 F F F <--- 22 440-459 192 192 192 341 341 341 F F F 23 460-479 185 185 185 328 328 328 F T T <--- 24 480-499 190 190 190 337 337 337 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 190 192 192 338 342 342 F F F <--- 27 540-559 190 192 192 337 342 342 F F F <--- 28 560-579 188 192 192 335 344 344 F T T <--- 29 580-599 188 188 188 335 335 335 F F F Max gen: 192, Max flush: 344 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 192 192 192 344 344 344 F F F 3 060-079 192 192 192 344 344 344 F F F 4 080-099 190 190 190 338 338 338 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 193 193 193 345 345 345 F F F 7 140-159 190 190 190 337 337 337 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 184 184 184 326 326 326 F F F 10 200-219 187 187 187 334 334 334 F F F 11 220-239 192 192 192 344 344 344 F F F 12 240-259 193 193 193 345 345 345 F F F 13 260-279 193 193 193 345 345 345 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 184 184 326 326 326 F F F 16 320-339 193 193 193 345 345 345 F F F 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 192 192 192 344 344 344 F F F 20 400-419 192 192 192 344 344 344 F F F 21 420-439 192 192 192 344 344 344 F F F 22 440-459 192 192 192 341 341 341 F F F 23 460-479 193 193 193 345 345 345 F F F 24 480-499 190 190 190 337 337 337 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 192 192 192 342 342 342 F F F 28 560-579 193 193 193 345 345 345 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 193, Max flush: 345 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.719353397Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:57.730035827Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.730094583Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74038712Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.740685779Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.740724957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.740737196Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.740754665Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"Crucible 113ef905-8c2c-4859-af5f-be9c9b3d2cec has session id: 5ee72925-837f-40aa-b3fe-f393af161bc6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741094591Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Upstairs opts: Upstairs UUID: 113ef905-8c2c-4859-af5f-be9c9b3d2cec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741141028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Crucible stats registered with UUID: 113ef905-8c2c-4859-af5f-be9c9b3d2cec","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741156887Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:57.741170346Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741359893Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741397341Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74141299Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741515273Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741533111Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"113ef905-8c2c-4859-af5f-be9c9b3d2cec active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74154968Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741562609Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741576978Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741590858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741604347Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741618496Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741641184Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42358)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741679641Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48106)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741828781Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48128)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.741855869Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742397212Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742417871Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74243425Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742790966Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742840662Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[0]R flush_numbers[0..12]: [338, 335, 344, 344, 338, 338, 345, 337, 344, 326, 334, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742912137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[0]R generation[0..12]: [190, 188, 192, 192, 190, 190, 193, 190, 192, 184, 187, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742929296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742946065Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[1]R flush_numbers[0..12]: [338, 335, 344, 344, 338, 338, 345, 337, 344, 326, 334, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742961994Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[1]R generation[0..12]: [190, 188, 192, 192, 190, 190, 193, 190, 192, 184, 187, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.742978093Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74301109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[2]R flush_numbers[0..12]: [338, 335, 344, 344, 338, 338, 345, 337, 344, 326, 334, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743027119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[2]R generation[0..12]: [190, 188, 192, 192, 190, 190, 193, 190, 192, 184, 187, 192]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743042628Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:38:57.743058957Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743077296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743091565Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743105074Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743118533Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"":"downstairs","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743132882Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743145541Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"0","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.74316007Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"1","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743177059Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"client":"2","":"downstairs","session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743191448Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743205027Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"113ef905-8c2c-4859-af5f-be9c9b3d2cec is now active with session: 5ee72925-837f-40aa-b3fe-f393af161bc6","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743220036Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743234725Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982,"session_id":"5ee72925-837f-40aa-b3fe-f393af161bc6"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743248394Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} {"msg":"Activated sub_volume 113ef905-8c2c-4859-af5f-be9c9b3d2cec","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:38:57.743260013Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 231 len 8192 data: 21 22 02/30 Read block 549 len 12288 03/30 Read block 280 len 24576 04/30 Write block 337 len 8192 data: 16 17 05/30 Write block 532 len 32768 data: 14 15 16 12 13 17 15 15 06/30 Read block 497 len 40960 07/30 Read block 415 len 32768 08/30 Write block 79 len 8192 data: 12 13 09/30 Read block 491 len 16384 10/30 Write block 125 len 12288 data: 13 16 15 11/30 Write block 534 len 40960 data: 17 13 14 18 16 16 17 20 22 19 12/30 Read block 423 len 36864 13/30 Read block 323 len 16384 14/30 Read block 412 len 20480 15/30 Flush 16/30 Write block 566 len 40960 data: 15 13 12 9 13 13 10 8 11 12 17/30 Write block 314 len 28672 data: 17 18 21 23 20 18 16 18/30 Read block 411 len 32768 19/30 Write block 38 len 4096 data: 22 20/30 Write block 475 len 40960 data: 8 8 13 17 18 20 19 18 16 11 21/30 Flush 22/30 Write block 141 len 40960 data: 19 15 13 12 11 9 8 10 11 10 23/30 Write block 182 len 20480 data: 17 15 14 13 11 24/30 Read block 162 len 12288 25/30 Read block 31 len 32768 26/30 Read block 226 len 40960 27/30 Write block 403 len 36864 data: 19 20 19 18 19 19 19 18 17 28/30 Read block 547 len 24576 29/30 Write block 432 len 36864 data: 17 15 15 16 14 15 16 11 10 30/30 Write block 204 len 12288 data: 18 18 19 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:38:57.756681896Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1982} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 188 194 194 335 347 347 F F F <--- 2 040-059 192 192 192 344 344 344 F F F 3 060-079 192 194 194 344 346 346 F F F <--- 4 080-099 190 194 194 338 346 346 F F F <--- 5 100-119 190 190 190 338 338 338 F F F 6 120-139 193 194 194 345 346 346 F F F <--- 7 140-159 190 190 190 337 337 337 F T T <--- 8 160-179 192 192 192 344 344 344 F F F 9 180-199 184 184 184 326 326 326 F T T <--- 10 200-219 187 187 187 334 334 334 F T T <--- 11 220-239 192 194 194 344 346 346 F F F <--- 12 240-259 193 193 193 345 345 345 F F F 13 260-279 193 193 193 345 345 345 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 184 194 194 326 347 347 F F F <--- 16 320-339 193 194 194 345 347 347 F F F <--- 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 192 192 192 344 344 344 F F F 20 400-419 192 192 192 344 344 344 F T T <--- 21 420-439 192 192 192 344 344 344 F T T <--- 22 440-459 192 192 192 341 341 341 F T T <--- 23 460-479 193 194 194 345 347 347 F F F <--- 24 480-499 190 194 194 337 347 347 F F F <--- 25 500-519 184 184 184 326 326 326 F F F 26 520-539 192 194 194 342 346 346 F F F <--- 27 540-559 192 194 194 342 346 346 F F F <--- 28 560-579 193 194 194 345 347 347 F F F <--- 29 580-599 188 188 188 335 335 335 F F F Max gen: 194, Max flush: 347 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 194 194 194 347 347 347 F F F 2 040-059 192 192 192 344 344 344 F F F 3 060-079 194 194 194 346 346 346 F F F 4 080-099 194 194 194 346 346 346 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 194 194 194 346 346 346 F F F 7 140-159 195 195 195 348 348 348 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 195 195 195 348 348 348 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 194 194 194 346 346 346 F F F 12 240-259 193 193 193 345 345 345 F F F 13 260-279 193 193 193 345 345 345 F F F 14 280-299 184 184 184 326 326 326 F F F 15 300-319 194 194 194 347 347 347 F F F 16 320-339 194 194 194 347 347 347 F F F 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 192 192 192 344 344 344 F F F 20 400-419 195 195 195 348 348 348 F F F 21 420-439 195 195 195 348 348 348 F F F 22 440-459 195 195 195 348 348 348 F F F 23 460-479 194 194 194 347 347 347 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 184 184 184 326 326 326 F F F 26 520-539 194 194 194 346 346 346 F F F 27 540-559 194 194 194 346 346 346 F F F 28 560-579 194 194 194 347 347 347 F F F 29 580-599 188 188 188 335 335 335 F F F Max gen: 195, Max flush: 348 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.103512287Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.115352318Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.115672016Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.115711174Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.115723143Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.115740922Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"Crucible 7d8d587a-f881-4e28-a7d1-fd61cf318584 has session id: a27e9a2a-8abd-428f-a5ae-43feb5367d10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116112006Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Upstairs opts: Upstairs UUID: 7d8d587a-f881-4e28-a7d1-fd61cf318584, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116147704Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Crucible stats registered with UUID: 7d8d587a-f881-4e28-a7d1-fd61cf318584","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116164063Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:39:02.116177562Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116331121Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116360009Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116375088Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.11650046Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116518538Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"7d8d587a-f881-4e28-a7d1-fd61cf318584 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116534617Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116547416Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116560956Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116574695Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116594493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116608962Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116623401Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:63782)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116670188Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:56459)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116698266Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:53546)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.116816018Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"io task","client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.1173722Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117461404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117588715Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117760234Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117800721Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[0]R flush_numbers[0..12]: [338, 347, 344, 346, 346, 338, 346, 348, 344, 348, 348, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117951241Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[0]R generation[0..12]: [190, 194, 192, 194, 194, 190, 194, 195, 192, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117968119Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.117984688Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[1]R flush_numbers[0..12]: [338, 347, 344, 346, 346, 338, 346, 348, 344, 348, 348, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118000477Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[1]R generation[0..12]: [190, 194, 192, 194, 194, 190, 194, 195, 192, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118016186Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118032295Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[2]R flush_numbers[0..12]: [338, 347, 344, 346, 346, 338, 346, 348, 344, 348, 348, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118047854Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[2]R generation[0..12]: [190, 194, 192, 194, 194, 190, 194, 195, 192, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118063333Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118079182Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-16T21:39:02.118092921Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118113219Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118126719Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118140028Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"":"downstairs","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118153637Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118166386Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"0","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118180515Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"1","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118194404Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"client":"2","":"downstairs","session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118210353Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118223972Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"7d8d587a-f881-4e28-a7d1-fd61cf318584 is now active with session: a27e9a2a-8abd-428f-a5ae-43feb5367d10","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118239051Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.11825179Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990,"session_id":"a27e9a2a-8abd-428f-a5ae-43feb5367d10"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118265109Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} {"msg":"Activated sub_volume 7d8d587a-f881-4e28-a7d1-fd61cf318584","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:02.118276128Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 594 len 20480 data: 8 8 6 6 5 02/30 Read block 122 len 28672 03/30 Write block 288 len 40960 data: 15 13 11 13 11 12 11 10 12 14 04/30 Read block 138 len 8192 05/30 Write block 314 len 12288 data: 18 19 22 06/30 Read block 326 len 8192 07/30 Read block 176 len 24576 08/30 Read block 404 len 28672 09/30 Read block 532 len 20480 10/30 Read block 54 len 24576 11/30 Write block 20 len 20480 data: 14 16 13 14 18 12/30 Read block 494 len 28672 13/30 Write block 263 len 32768 data: 15 15 13 14 11 14 15 15 14/30 Read block 230 len 24576 15/30 Read block 531 len 28672 16/30 Read block 50 len 40960 17/30 Write block 238 len 36864 data: 12 14 13 17 20 18 15 14 17 18/30 Write block 389 len 20480 data: 17 16 14 15 20 19/30 Read block 405 len 40960 20/30 Write block 469 len 4096 data: 16 21/30 Read block 367 len 36864 22/30 Write block 514 len 24576 data: 12 15 17 20 21 18 23/30 Read block 135 len 4096 24/30 Write block 470 len 40960 data: 17 16 15 13 11 9 9 14 18 19 25/30 Read block 357 len 28672 26/30 Write block 245 len 28672 data: 15 18 13 15 14 13 12 27/30 Write block 583 len 24576 data: 15 15 13 14 12 8 28/30 Write block 330 len 28672 data: 13 15 15 14 14 18 17 29/30 Read block 13 len 16384 30/30 Read block 407 len 20480 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:39:02.131171217Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1990} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 194 194 194 347 347 347 T F T <--- 2 040-059 192 192 192 344 344 344 F F F 3 060-079 194 194 194 346 346 346 F F F 4 080-099 194 194 194 346 346 346 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 194 194 194 346 346 346 F F F 7 140-159 195 195 195 348 348 348 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 195 195 195 348 348 348 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 194 194 194 346 346 346 T F T <--- 12 240-259 193 193 193 345 345 345 T F T <--- 13 260-279 193 193 193 345 345 345 T F T <--- 14 280-299 184 184 184 326 326 326 T T T 15 300-319 194 194 194 347 347 347 T F T <--- 16 320-339 194 194 194 347 347 347 T F T <--- 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 192 192 192 344 344 344 T F T <--- 20 400-419 195 195 195 348 348 348 F F F 21 420-439 195 195 195 348 348 348 F F F 22 440-459 195 195 195 348 348 348 F F F 23 460-479 194 194 194 347 347 347 T F T <--- 24 480-499 194 194 194 347 347 347 F F F 25 500-519 184 184 184 326 326 326 T F T <--- 26 520-539 194 194 194 346 346 346 F F F 27 540-559 194 194 194 346 346 346 F F F 28 560-579 194 194 194 347 347 347 F F F 29 580-599 188 188 188 335 335 335 T T T Max gen: 195, Max flush: 348 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F F F 1 020-039 196 196 196 349 349 349 F F F 2 040-059 192 192 192 344 344 344 F F F 3 060-079 194 194 194 346 346 346 F F F 4 080-099 194 194 194 346 346 346 F F F 5 100-119 190 190 190 338 338 338 F F F 6 120-139 194 194 194 346 346 346 F F F 7 140-159 195 195 195 348 348 348 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 195 195 195 348 348 348 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 196 196 196 349 349 349 F F F 13 260-279 196 196 196 349 349 349 F F F 14 280-299 196 196 196 349 349 349 F F F 15 300-319 196 196 196 349 349 349 F F F 16 320-339 196 196 196 349 349 349 F F F 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 196 196 196 349 349 349 F F F 20 400-419 195 195 195 348 348 348 F F F 21 420-439 195 195 195 348 348 348 F F F 22 440-459 195 195 195 348 348 348 F F F 23 460-479 196 196 196 349 349 349 F F F 24 480-499 194 194 194 347 347 347 F F F 25 500-519 196 196 196 349 349 349 F F F 26 520-539 194 194 194 346 346 346 F F F 27 540-559 194 194 194 346 346 346 F F F 28 560-579 194 194 194 347 347 347 F F F 29 580-599 196 196 196 349 349 349 F F F Max gen: 196, Max flush: 349 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.454360774Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:39:06.46510858Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.465163456Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.475599133Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.475903952Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"4d4cc95d87dca59ffda6995ddca82a3fb397461e\",\n git_commit_timestamp: \"2026-03-16T21:12:27.000000000Z\",\n git_branch: \"alan/downstairs-dump-tolerant\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.475941969Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.475953919Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.475971457Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"Crucible 90a7ca51-0252-47ac-a1cc-28899a9c0ef5 has session id: f6ab748f-4e58-4afb-854d-83c0e2d406bf","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476330303Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Upstairs opts: Upstairs UUID: 90a7ca51-0252-47ac-a1cc-28899a9c0ef5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476364901Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Crucible stats registered with UUID: 90a7ca51-0252-47ac-a1cc-28899a9c0ef5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476380309Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:39:06.476393619Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476576126Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476612364Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476627993Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476720876Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476739495Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"90a7ca51-0252-47ac-a1cc-28899a9c0ef5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476756264Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476771423Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476785592Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476799551Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.47681324Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476835288Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476849767Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:35037)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476864526Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58317)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476879415Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:39322)","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.476894234Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"io task","client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.477558979Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.477649443Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.477774064Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.477939863Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.477971281Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[0]R flush_numbers[0..12]: [338, 349, 344, 346, 346, 338, 346, 348, 344, 348, 348, 349]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478140989Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[0]R generation[0..12]: [190, 196, 192, 194, 194, 190, 194, 195, 192, 195, 195, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478159558Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478178967Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[1]R flush_numbers[0..12]: [338, 349, 344, 346, 346, 338, 346, 348, 344, 348, 348, 349]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478195045Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[1]R generation[0..12]: [190, 196, 192, 194, 194, 190, 194, 195, 192, 195, 195, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478210944Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478227493Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[2]R flush_numbers[0..12]: [338, 349, 344, 346, 346, 338, 346, 348, 344, 348, 348, 349]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478243342Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[2]R generation[0..12]: [190, 196, 192, 194, 194, 190, 194, 195, 192, 195, 195, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478259011Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.47827547Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"}Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 197","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478295199Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Generation requested: 198 >= found:197","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478309508Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478322897Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478336296Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"":"downstairs","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478350445Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478363254Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"0","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-16T21:39:06.478377733Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"1","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478396582Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"client":"2","":"downstairs","session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478410801Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.47842425Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"90a7ca51-0252-47ac-a1cc-28899a9c0ef5 is now active with session: f6ab748f-4e58-4afb-854d-83c0e2d406bf","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478439069Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478451618Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998,"session_id":"f6ab748f-4e58-4afb-854d-83c0e2d406bf"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478465137Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} {"msg":"Activated sub_volume 90a7ca51-0252-47ac-a1cc-28899a9c0ef5","v":0,"name":"crucible","level":30,"time":"2026-03-16T21:39:06.478476256Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 28672 data: 18 17 15 16 21 20 19 02/30 Read block 537 len 4096 03/30 Read block 280 len 4096 04/30 Flush 05/30 Write block 314 len 28672 data: 19 20 23 24 21 19 17 06/30 Read block 293 len 28672 07/30 Write block 431 len 16384 data: 18 18 16 16 08/30 Write block 437 len 36864 data: 16 17 12 11 13 14 14 11 13 09/30 Write block 165 len 4096 data: 16 10/30 Flush 11/30 Write block 101 len 8192 data: 15 14 12/30 Write block 498 len 16384 data: 17 17 17 19 13/30 Write block 0 len 24576 data: 2 6 8 11 10 12 14/30 Read block 432 len 24576 15/30 Write block 90 len 20480 data: 10 11 9 11 12 16/30 Write block 511 len 24576 data: 14 14 11 13 16 18 17/30 Write block 3 len 36864 data: 12 11 13 14 14 13 17 18 17 18/30 Write block 56 len 4096 data: 12 19/30 Read block 284 len 24576 20/30 Read block 269 len 12288 21/30 Read block 42 len 16384 22/30 Read block 259 len 16384 23/30 Write block 326 len 24576 data: 10 11 11 14 14 16 24/30 Write block 532 len 4096 data: 15 25/30 Write block 4 len 24576 data: 12 14 15 15 14 18 26/30 Read block 324 len 28672 27/30 Read block 303 len 20480 28/30 Read block 354 len 40960 29/30 Read block 194 len 40960 30/30 Write block 304 len 20480 data: 12 10 10 12 12 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-16T21:39:06.489926034Z","hostname":"w-01KKW8WG4PSGGHQQYBKH64M0X3","pid":1998} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 338 338 338 F T T <--- 1 020-039 196 196 196 349 349 349 F F F 2 040-059 192 192 192 344 344 344 F T T <--- 3 060-079 194 194 194 346 346 346 F F F 4 080-099 194 194 194 346 346 346 F T T <--- 5 100-119 190 190 190 338 338 338 F T T <--- 6 120-139 194 194 194 346 346 346 F F F 7 140-159 195 195 195 348 348 348 F F F 8 160-179 192 198 198 344 351 351 F F F <--- 9 180-199 195 195 195 348 348 348 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 196 196 196 349 349 349 F F F 13 260-279 196 196 196 349 349 349 F F F 14 280-299 196 196 196 349 349 349 F F F 15 300-319 196 198 198 349 351 351 F T T <--- 16 320-339 196 198 198 349 351 351 F T T <--- 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 196 198 198 349 350 350 T F F <--- 20 400-419 195 195 195 348 348 348 F F F 21 420-439 195 198 198 348 351 351 F F F <--- 22 440-459 195 198 198 348 351 351 F F F <--- 23 460-479 196 196 196 349 349 349 F F F 24 480-499 194 194 194 347 347 347 F T T <--- 25 500-519 196 196 196 349 349 349 F T T <--- 26 520-539 194 194 194 346 346 346 F T T <--- 27 540-559 194 194 194 346 346 346 F F F 28 560-579 194 194 194 347 347 347 F F F 29 580-599 196 196 196 349 349 349 F F F Max gen: 198, Max flush: 351 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 352 352 352 F F F 1 020-039 196 196 196 349 349 349 F F F 2 040-059 199 199 199 352 352 352 F F F 3 060-079 194 194 194 346 346 346 F F F 4 080-099 199 199 199 352 352 352 F F F 5 100-119 199 199 199 352 352 352 F F F 6 120-139 194 194 194 346 346 346 F F F 7 140-159 195 195 195 348 348 348 F F F 8 160-179 198 198 198 351 351 351 F F F 9 180-199 195 195 195 348 348 348 F F F 10 200-219 195 195 195 348 348 348 F F F 11 220-239 196 196 196 349 349 349 F F F 12 240-259 196 196 196 349 349 349 F F F 13 260-279 196 196 196 349 349 349 F F F 14 280-299 196 196 196 349 349 349 F F F 15 300-319 199 199 199 352 352 352 F F F 16 320-339 199 199 199 352 352 352 F F F 17 340-359 193 193 193 345 345 345 F F F 18 360-379 192 192 192 344 344 344 F F F 19 380-399 198 198 198 350 350 350 F F F 20 400-419 195 195 195 348 348 348 F F F 21 420-439 198 198 198 351 351 351 F F F 22 440-459 198 198 198 351 351 351 F F F 23 460-479 196 196 196 349 349 349 F F F 24 480-499 199 199 199 352 352 352 F F F 25 500-519 199 199 199 352 352 352 F F F 26 520-539 199 199 199 352 352 352 F F F 27 540-559 194 194 194 346 346 346 F F F 28 560-579 194 194 194 347 347 347 F F F 29 580-599 196 196 196 349 349 349 F F F Max gen: 199, Max flush: 352 7:36 Test duration Test completed real 7:36.263821002 user 36.316577094 sys 17.200504578 trap 0.015238700 tflt 0.074259678 dflt 0.065130405 kflt 0.003373947 lock 12:02:01.739287164 slp 36:51.559224594 lat 1.375840519 stop 0.034302303