Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26411957 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.016456913 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.015818503 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1201 1202 1203 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.884457753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.905741175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.908522494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.908649051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.908663256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.908681661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"Crucible b41ff823-c50c-4779-9bc2-85d2a9575a34 has session id: 7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.909604246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Upstairs opts: Upstairs UUID: b41ff823-c50c-4779-9bc2-85d2a9575a34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.909744978Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Crucible stats registered with UUID: b41ff823-c50c-4779-9bc2-85d2a9575a34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.909762413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:13.909811958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910386929Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910428111Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910443476Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910801683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"task":"crutest"} {"msg":"b41ff823-c50c-4779-9bc2-85d2a9575a34 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.91085939Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910873814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910889069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910903063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910916827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910931331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910946176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.910956739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51857)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.911557628Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42497)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.911586477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57704)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.911602821Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.913795385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.913840468Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.91387933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.914837435Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.9149202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.914936955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.91495378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.914968914Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.914983979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915000304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915015148Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915030123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915046387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915059992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915074206Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.91508721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915100544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"downstairs","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915116538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915129342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915143426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"1","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.91515713Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"client":"2","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915171515Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915185289Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"b41ff823-c50c-4779-9bc2-85d2a9575a34 is now active with session: 7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915200273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915212577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915225961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"Activated sub_volume b41ff823-c50c-4779-9bc2-85d2a9575a34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:13.915562791Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:17.983534901Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:17.983908373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:17.983980844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:17.984064179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:17.984084075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"7a481a4a-cdc1-4e4b-8ec5-8f60f589de8b"} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.011080479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.022936211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023269971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023307512Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023319856Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023337631Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"Crucible 241663f5-bb9d-43d7-a8f2-8ca73380821b has session id: 271f85f4-3d64-40a9-9960-5682da8bf0d3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023679023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Upstairs opts: Upstairs UUID: 241663f5-bb9d-43d7-a8f2-8ca73380821b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023714863Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Crucible stats registered with UUID: 241663f5-bb9d-43d7-a8f2-8ca73380821b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023730418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:18.023747583Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023898698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023933018Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.023948383Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024084483Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024099418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"241663f5-bb9d-43d7-a8f2-8ca73380821b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024116293Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024129307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024143731Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024157215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024170719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024185183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024199838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39667)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024226216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56205)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024242671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47906)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024257455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024924274Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.024963725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.025614219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.02595159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.025975557Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026002915Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.02601952Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026034925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026049859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026066164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026081298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026096143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026112348Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026125932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026148298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026162032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026175646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"":"downstairs","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026189971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026202725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"0","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026216669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"client":"1","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026230873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65PDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","pid":1208,"client":"2","":"downstairs","session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026248928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026262452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"241663f5-bb9d-43d7-a8f2-8ca73380821b is now active with session: 271f85f4-3d64-40a9-9960-5682da8bf0d3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026277577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026290501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208,"session_id":"271f85f4-3d64-40a9-9960-5682da8bf0d3"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026303665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} {"msg":"Activated sub_volume 241663f5-bb9d-43d7-a8f2-8ca73380821b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:18.026314848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 24576 data: 2 2 2 2 2 2 02/30 Read block 246 len 24576 03/30 Flush 04/30 Write block 282 len 20480 data: 2 2 2 2 2 05/30 Read block 282 len 4096 06/30 Read block 429 len 32768 07/30 Flush 08/30 Write block 32 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Read block 282 len 24576 10/30 Read block 216 len 32768 11/30 Write block 296 len 4096 data: 2 12/30 Read block 38 len 32768 13/30 Write block 29 len 8192 data: 2 2 14/30 Read block 282 len 12288 15/30 Write block 394 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Read block 486 len 8192 17/30 Read block 534 len 16384 18/30 Flush 19/30 Write block 173 len 16384 data: 2 2 2 2 20/30 Read block 165 len 28672 21/30 Flush 22/30 Write block 569 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 319 len 28672 24/30 Flush 25/30 Write block 544 len 20480 data: 2 2 2 2 2 26/30 Read block 43 len 8192 27/30 Write block 262 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 136 len 40960 29/30 Read block 271 len 16384 30/30 Write block 546 len 24576 data: 3 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:18.039673378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 1 2 4 1 4 F F F <--- 2 040-059 2 1 2 4 1 4 F F F <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 1 2 5 1 5 F F F <--- 9 180-199 2 1 2 2 1 2 F T F <--- 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 T F T <--- 14 280-299 2 1 2 4 1 4 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 1 2 4 1 4 F F F <--- 20 400-419 2 1 2 4 1 4 F F F <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T F T <--- 28 560-579 2 1 2 6 1 6 F F F <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 3 3 3 7 7 7 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 7 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.534815239Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.547216374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.547588664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.547633508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.547646282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.547668788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"Crucible 638b0b92-c07d-4bfd-9a34-3fe4623e2619 has session id: 6262d51f-6244-4fca-bdd9-7982c89edcf7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548114611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Upstairs opts: Upstairs UUID: 638b0b92-c07d-4bfd-9a34-3fe4623e2619, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548170988Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Crucible stats registered with UUID: 638b0b92-c07d-4bfd-9a34-3fe4623e2619","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548187793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:32.548201187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548390653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548426454Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548442109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548618371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548636507Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"638b0b92-c07d-4bfd-9a34-3fe4623e2619 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548652331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548665265Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.54867954Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548694084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548707878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548725503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548740178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40862)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.548836706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44763)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.549008297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36402)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.549031945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.549543057Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.549637845Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.549693062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550062822Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[0]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 1, 5, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.55009023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550105955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.55012264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[1]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 1, 5, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550137894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550153099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550173845Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[2]R flush_numbers[0..12]: [1, 4, 4, 1, 1, 1, 1, 1, 5, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550189259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550204224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550220309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550233733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550247997Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550261341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:12:32.550274835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"":"downstairs","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550295011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550307915Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"0","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550322299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"1","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550336403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"client":"2","":"downstairs","session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550351258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550364682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"638b0b92-c07d-4bfd-9a34-3fe4623e2619 is now active with session: 6262d51f-6244-4fca-bdd9-7982c89edcf7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550379606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.55039237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217,"session_id":"6262d51f-6244-4fca-bdd9-7982c89edcf7"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550411276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} {"msg":"Activated sub_volume 638b0b92-c07d-4bfd-9a34-3fe4623e2619","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:32.550434522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 404 len 24576 03/30 Read block 421 len 28672 04/30 Read block 401 len 8192 05/30 Write block 267 len 32768 data: 3 3 2 2 2 2 2 2 06/30 Write block 113 len 12288 data: 2 2 2 07/30 Read block 307 len 4096 08/30 Write block 284 len 36864 data: 3 3 3 2 2 2 2 2 2 09/30 Write block 271 len 12288 data: 3 3 3 10/30 Flush 11/30 Write block 346 len 20480 data: 3 3 3 3 3 12/30 Read block 227 len 12288 13/30 Read block 354 len 40960 14/30 Read block 588 len 16384 15/30 Read block 401 len 32768 16/30 Write block 466 len 20480 data: 2 2 2 2 2 17/30 Write block 238 len 28672 data: 2 2 2 2 2 2 2 18/30 Write block 469 len 32768 data: 3 3 2 2 2 2 2 2 19/30 Write block 121 len 4096 data: 2 20/30 Write block 208 len 8192 data: 2 2 21/30 Read block 126 len 4096 22/30 Write block 122 len 28672 data: 2 2 2 2 2 2 2 23/30 Write block 378 len 8192 data: 2 2 24/30 Write block 289 len 28672 data: 3 3 3 3 2 2 2 25/30 Flush 26/30 Write block 14 len 8192 data: 2 2 27/30 Read block 125 len 8192 28/30 Write block 75 len 20480 data: 2 2 2 2 2 29/30 Write block 517 len 8192 data: 2 2 30/30 Write block 185 len 16384 data: 2 2 2 3 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 30 3 36 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:32.562070259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1217} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T F T <--- 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 1 1 1 1 1 1 T F T <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 1 4 8 1 8 F F F <--- 6 120-139 4 1 4 9 1 9 F F F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 2 2 2 2 2 2 T F T <--- 10 200-219 4 1 4 9 1 9 F F F <--- 11 220-239 4 1 4 9 1 9 F F F <--- 12 240-259 4 1 4 9 1 9 F F F <--- 13 260-279 4 3 4 8 7 8 F F F <--- 14 280-299 4 2 4 9 4 9 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 1 4 9 1 9 F F F <--- 18 360-379 4 1 4 9 1 9 F F F <--- 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 1 4 9 1 9 F F F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 T F T <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 10 10 10 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 5 5 5 10 10 10 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 4 4 4 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 5 5 5 10 10 10 F F F 10 200-219 4 4 4 9 9 9 F F F 11 220-239 4 4 4 9 9 9 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 9 9 9 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 9 9 9 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.944097615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:46.954834795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.954890941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.965335834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.965617448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.965656209Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.965668233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.965686358Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"Crucible 1d9e2073-c8c2-4857-b6f1-5c241c3742e0 has session id: ce8bd1b0-66c0-455c-88b0-829784b55d1b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966039323Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Upstairs opts: Upstairs UUID: 1d9e2073-c8c2-4857-b6f1-5c241c3742e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966066001Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Crucible stats registered with UUID: 1d9e2073-c8c2-4857-b6f1-5c241c3742e0","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966081136Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:46.9660944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966245295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966269202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966284216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966448665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96646556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"1d9e2073-c8c2-4857-b6f1-5c241c3742e0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966482115Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966495109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966509383Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966523257Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966537012Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966551466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96656538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58629)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966580524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57201)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966595789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42639)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.966738592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96730497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967344312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967361217Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967733298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[0]R flush_numbers[0..12]: [10, 4, 4, 10, 1, 8, 9, 1, 5, 10, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967754904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[0]R generation[0..12]: [5, 2, 2, 5, 1, 4, 4, 1, 2, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967770129Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967786634Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[1]R flush_numbers[0..12]: [10, 4, 4, 10, 1, 8, 9, 1, 5, 10, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967801828Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[1]R generation[0..12]: [5, 2, 2, 5, 1, 4, 4, 1, 2, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967816723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967833118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[2]R flush_numbers[0..12]: [10, 4, 4, 10, 1, 8, 9, 1, 5, 10, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967848042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[2]R generation[0..12]: [5, 2, 2, 5, 1, 4, 4, 1, 2, 5, 4, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967862767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967879121Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967892345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96790643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967929016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96794255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"":"downstairs","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967956865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967969808Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"0","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967984203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"1","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.967997937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"client":"2","":"downstairs","session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.968012241Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-03-17T14:12:46.968028246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"1d9e2073-c8c2-4857-b6f1-5c241c3742e0 is now active with session: ce8bd1b0-66c0-455c-88b0-829784b55d1b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.96804353Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.968056664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226,"session_id":"ce8bd1b0-66c0-455c-88b0-829784b55d1b"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.968070158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} {"msg":"Activated sub_volume 1d9e2073-c8c2-4857-b6f1-5c241c3742e0","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:12:46.968081712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 20480 data: 2 2 2 2 2 02/30 Read block 554 len 28672 03/30 Read block 387 len 40960 04/30 Read block 268 len 20480 05/30 Read block 157 len 32768 06/30 Read block 53 len 36864 07/30 Read block 224 len 4096 08/30 Read block 557 len 16384 09/30 Read block 57 len 12288 10/30 Write block 529 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Read block 452 len 24576 12/30 Flush 13/30 Write block 197 len 32768 data: 2 2 2 2 2 2 2 2 14/30 Read block 340 len 36864 15/30 Read block 173 len 28672 16/30 Read block 474 len 16384 17/30 Write block 443 len 16384 data: 2 2 2 2 18/30 Write block 102 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Read block 436 len 16384 20/30 Flush 21/30 Write block 18 len 8192 data: 2 2 22/30 Read block 458 len 32768 23/30 Read block 54 len 36864 24/30 Read block 529 len 16384 25/30 Write block 250 len 8192 data: 2 2 26/30 Write block 235 len 40960 data: 2 2 2 3 3 3 3 3 3 3 27/30 Write block 292 len 32768 data: 4 3 3 3 3 2 2 2 28/30 Read block 505 len 20480 29/30 Read block 527 len 28672 30/30 Read block 50 len 4096 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:12:46.98099586Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1226} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 10 10 10 F T T <--- 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 5 6 6 10 11 11 T F F <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 4 6 6 8 12 12 F F F <--- 6 120-139 4 4 4 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 5 6 6 10 12 12 F F F <--- 10 200-219 4 6 6 9 12 12 F F F <--- 11 220-239 4 4 4 9 9 9 F T T <--- 12 240-259 4 4 4 9 9 9 F T T <--- 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 9 9 9 F T T <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 9 9 9 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 6 6 1 12 12 F F F <--- 23 460-479 4 4 4 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 1 6 6 1 11 11 F F F <--- 27 540-559 3 3 3 7 7 7 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 12 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 6 6 6 12 12 12 F F F 6 120-139 4 4 4 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 6 6 6 12 12 12 F F F 11 220-239 7 7 7 13 13 13 F F F 12 240-259 7 7 7 13 13 13 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 7 7 7 13 13 13 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 9 9 9 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 12 12 12 F F F 23 460-479 4 4 4 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 13 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.306528534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.318742604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319114485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319152796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.31916568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319184145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"Crucible ed7b4abe-fb16-407a-817e-55e42c241836 has session id: d709981f-942b-4d94-893b-87d3bab44785","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319555746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Upstairs opts: Upstairs UUID: ed7b4abe-fb16-407a-817e-55e42c241836, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319596028Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Crucible stats registered with UUID: ed7b4abe-fb16-407a-817e-55e42c241836","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319611103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:01.319624326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319815173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319850314Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319865818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.319986534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.32000472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"ed7b4abe-fb16-407a-817e-55e42c241836 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320035569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320049733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320064398Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320078342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320092226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320107Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320124255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"ds_connection connected from Ok(127.0.0.1:59561)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320187854Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"ds_connection connected from Ok(127.0.0.1:56556)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320215943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"ds_connection connected from Ok(127.0.0.1:33733)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320232118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.32091202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320947401Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.320993414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321398765Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[0]R flush_numbers[0..12]: [13, 4, 4, 11, 1, 12, 9, 1, 5, 12, 12, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321421812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[0]R generation[0..12]: [7, 2, 2, 6, 1, 6, 4, 1, 2, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321437387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321454072Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[1]R flush_numbers[0..12]: [13, 4, 4, 11, 1, 12, 9, 1, 5, 12, 12, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321469486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[1]R generation[0..12]: [7, 2, 2, 6, 1, 6, 4, 1, 2, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321484471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321501046Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[2]R flush_numbers[0..12]: [13, 4, 4, 11, 1, 12, 9, 1, 5, 12, 12, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321519031Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[2]R generation[0..12]: [7, 2, 2, 6, 1, 6, 4, 1, 2, 6, 6, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321533925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.32155024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321563814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321578169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321598195Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321611849Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"":"downstairs","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321626253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321638857Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"0","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321652861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"1","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321667055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"client":"2","":"downstairs","session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321681399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321697864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"ed7b4abe-fb16-407a-817e-55e42c241836 is now active with session: d709981f-942b-4d94-893b-87d3bab44785","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321713049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321725633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235,"session_id":"d709981f-942b-4d94-893b-87d3bab44785"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321740667Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} {"msg":"Activated sub_volume ed7b4abe-fb16-407a-817e-55e42c241836","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:01.321752491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 24576 data: 2 2 2 2 2 2 02/30 Read block 269 len 28672 03/30 Flush 04/30 Write block 246 len 20480 data: 2 2 2 2 3 05/30 Write block 294 len 16384 data: 4 4 4 3 06/30 Write block 451 len 12288 data: 2 2 2 07/30 Flush 08/30 Write block 334 len 32768 data: 2 2 2 2 2 2 2 3 09/30 Read block 54 len 16384 10/30 Read block 312 len 24576 11/30 Write block 78 len 24576 data: 3 3 2 2 2 2 12/30 Read block 324 len 8192 13/30 Write block 62 len 36864 data: 2 2 2 2 3 3 3 3 3 14/30 Write block 257 len 24576 data: 2 2 2 2 2 3 15/30 Read block 334 len 28672 16/30 Read block 3 len 28672 17/30 Read block 392 len 32768 18/30 Write block 182 len 16384 data: 2 2 2 3 19/30 Read block 284 len 32768 20/30 Flush 21/30 Write block 350 len 32768 data: 4 2 2 2 2 2 2 2 22/30 Flush 23/30 Write block 334 len 24576 data: 3 3 3 3 3 3 24/30 Write block 558 len 4096 data: 2 25/30 Read block 233 len 32768 26/30 Flush 27/30 Write block 500 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 520 len 12288 29/30 Write block 269 len 40960 data: 3 3 4 4 4 3 2 2 2 2 30/30 Write block 203 len 28672 data: 3 3 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 25 1 25 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1002)) Some(JobId(1025)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:01.334564248Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1235} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 6 8 16 11 16 F F F <--- 4 080-099 8 1 8 16 1 16 F F F <--- 5 100-119 6 6 6 12 12 12 F F F 6 120-139 4 4 4 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 8 6 8 16 12 16 F F F <--- 10 200-219 6 6 6 12 12 12 T F T <--- 11 220-239 7 7 7 13 13 13 F F F 12 240-259 8 7 8 16 13 16 F T F <--- 13 260-279 8 4 8 16 8 16 T F T <--- 14 280-299 8 7 8 15 13 15 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 8 1 8 18 1 18 F F F <--- 17 340-359 8 4 8 17 9 17 F F F <--- 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 8 6 8 15 12 15 F F F <--- 23 460-479 4 4 4 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 8 8 8 14 14 14 T F T <--- 26 520-539 6 6 6 11 11 11 F F F 27 540-559 8 3 8 18 7 18 F F F <--- 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 16 16 16 F F F 5 100-119 6 6 6 12 12 12 F F F 6 120-139 4 4 4 9 9 9 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 8 8 8 16 16 16 F F F 10 200-219 9 9 9 19 19 19 F F F 11 220-239 7 7 7 13 13 13 F F F 12 240-259 8 8 8 16 16 16 F F F 13 260-279 9 9 9 19 19 19 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 8 8 8 18 18 18 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 4 4 4 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 8 8 8 18 18 18 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.653015902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.664804965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665131872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665170013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665182687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665200753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"Crucible 8ab42cc9-ebc0-4741-841a-d841ccfa0c46 has session id: 931f829f-6b7a-40b3-8435-e7d9788029dc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665533712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Upstairs opts: Upstairs UUID: 8ab42cc9-ebc0-4741-841a-d841ccfa0c46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.66556138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Crucible stats registered with UUID: 8ab42cc9-ebc0-4741-841a-d841ccfa0c46","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665576655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:15.665590019Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665764921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665789548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665804402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665983076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.665999811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"8ab42cc9-ebc0-4741-841a-d841ccfa0c46 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666034551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666048775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666062789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666076653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666109933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666124418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666143023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:55828)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666283395Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33319)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666302841Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34302)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.666324047Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667023596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667076061Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667116643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667454514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[0]R flush_numbers[0..12]: [13, 4, 4, 16, 16, 12, 9, 1, 5, 16, 19, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667477101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[0]R generation[0..12]: [7, 2, 2, 8, 8, 6, 4, 1, 2, 8, 9, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667495656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667512321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[1]R flush_numbers[0..12]: [13, 4, 4, 16, 16, 12, 9, 1, 5, 16, 19, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667527536Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[1]R generation[0..12]: [7, 2, 2, 8, 8, 6, 4, 1, 2, 8, 9, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.66754227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667558795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[2]R flush_numbers[0..12]: [13, 4, 4, 16, 16, 12, 9, 1, 5, 16, 19, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.66757385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[2]R generation[0..12]: [7, 2, 2, 8, 8, 6, 4, 1, 2, 8, 9, 7]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667588504Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667604819Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667618413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667632467Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:13:15.667645921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667665107Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"":"downstairs","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667679501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667692375Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"0","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.66770829Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"1","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667722094Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"client":"2","":"downstairs","session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667736578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667749792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"8ab42cc9-ebc0-4741-841a-d841ccfa0c46 is now active with session: 931f829f-6b7a-40b3-8435-e7d9788029dc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667767917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667780361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244,"session_id":"931f829f-6b7a-40b3-8435-e7d9788029dc"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667793715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} {"msg":"Activated sub_volume 8ab42cc9-ebc0-4741-841a-d841ccfa0c46","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:15.667805329Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 586 len 12288 data: 2 2 2 02/30 Read block 418 len 40960 03/30 Write block 465 len 12288 data: 2 3 3 04/30 Write block 493 len 12288 data: 2 2 2 05/30 Read block 35 len 32768 06/30 Write block 237 len 36864 data: 3 4 4 4 4 4 4 4 2 07/30 Write block 432 len 24576 data: 2 2 2 2 2 2 08/30 Flush 09/30 Write block 96 len 40960 data: 2 2 2 2 2 2 3 3 3 3 10/30 Write block 543 len 40960 data: 2 3 3 4 4 4 3 3 3 2 11/30 Read block 307 len 24576 12/30 Read block 171 len 4096 13/30 Write block 483 len 4096 data: 2 14/30 Read block 317 len 4096 15/30 Read block 265 len 36864 16/30 Read block 339 len 24576 17/30 Write block 29 len 36864 data: 3 3 2 3 3 3 3 3 3 18/30 Read block 311 len 32768 19/30 Read block 395 len 4096 20/30 Flush 21/30 Flush 22/30 Write block 580 len 16384 data: 2 2 2 2 23/30 Read block 193 len 20480 24/30 Write block 93 len 40960 data: 2 2 2 3 3 3 3 3 3 4 25/30 Read block 376 len 12288 26/30 Read block 9 len 36864 27/30 Write block 283 len 12288 data: 3 4 4 28/30 Read block 534 len 32768 29/30 Write block 126 len 40960 data: 3 3 3 2 2 2 2 2 2 2 30/30 Write block 591 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:15.680103413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1244} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 10 2 10 21 4 21 F F F <--- 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 10 8 10 21 16 21 T F T <--- 5 100-119 10 6 10 21 12 21 T F T <--- 6 120-139 4 4 4 9 9 9 T F T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 8 8 8 16 16 16 F F F 10 200-219 9 9 9 19 19 19 F F F 11 220-239 10 7 10 20 13 20 F F F <--- 12 240-259 10 8 10 20 16 20 F F F <--- 13 260-279 9 9 9 19 19 19 F F F 14 280-299 8 8 8 15 15 15 T F T <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 8 8 8 18 18 18 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 10 1 10 20 1 20 F F F <--- 22 440-459 8 8 8 15 15 15 F F F 23 460-479 10 4 10 20 9 20 F T F <--- 24 480-499 10 1 10 21 1 21 F T F <--- 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 10 8 10 21 18 21 F F F <--- 28 560-579 2 2 2 6 6 6 F F F 29 580-599 10 1 10 20 1 20 T T T <--- Max gen: 10, Max flush: 21 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 10 10 10 21 21 21 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 11 11 11 22 22 22 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 5 5 5 F F F 9 180-199 8 8 8 16 16 16 F F F 10 200-219 9 9 9 19 19 19 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 9 9 9 19 19 19 F F F 14 280-299 11 11 11 22 22 22 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 8 8 8 18 18 18 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 10 10 10 21 21 21 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 2 2 2 6 6 6 F F F 29 580-599 11 11 11 22 22 22 F F F Max gen: 11, Max flush: 22 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.003065381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:30.013896038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.013950854Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.024520403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.024802277Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.024840859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.024852962Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.024871048Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"Crucible 2d70fd5e-9453-4868-8a26-171e2609ef2e has session id: bd9f7acf-feed-4253-b81f-50098ac2285f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025238647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Upstairs opts: Upstairs UUID: 2d70fd5e-9453-4868-8a26-171e2609ef2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025265985Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Crucible stats registered with UUID: 2d70fd5e-9453-4868-8a26-171e2609ef2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.02528132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:30.025294854Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025452741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025476118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025491133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025647209Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025665154Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"2d70fd5e-9453-4868-8a26-171e2609ef2e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025696044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025710148Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025724162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025738266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025752Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025766925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025781069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52928)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.02581734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33842)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.025979388Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45152)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.026007706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.026551568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.026579477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.026654889Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.026985548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[0]R flush_numbers[0..12]: [13, 21, 4, 16, 22, 22, 22, 1, 5, 16, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027007054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[0]R generation[0..12]: [7, 10, 2, 8, 11, 11, 11, 1, 2, 8, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027022769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027039454Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[1]R flush_numbers[0..12]: [13, 21, 4, 16, 22, 22, 22, 1, 5, 16, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027055069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[1]R generation[0..12]: [7, 10, 2, 8, 11, 11, 11, 1, 2, 8, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027070403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027087268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[2]R flush_numbers[0..12]: [13, 21, 4, 16, 22, 22, 22, 1, 5, 16, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027102853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[2]R generation[0..12]: [7, 10, 2, 8, 11, 11, 11, 1, 2, 8, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027121268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027138003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027151837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027171863Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027185908Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027199462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"":"downstairs","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027213836Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.02722649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"0","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027240924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"1","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027255198Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"client":"2","":"downstairs","session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2026-03-17T14:13:30.027269682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027286347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"2d70fd5e-9453-4868-8a26-171e2609ef2e is now active with session: bd9f7acf-feed-4253-b81f-50098ac2285f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027301522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027314136Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253,"session_id":"bd9f7acf-feed-4253-b81f-50098ac2285f"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.02732734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} {"msg":"Activated sub_volume 2d70fd5e-9453-4868-8a26-171e2609ef2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:30.027340654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 30 len 40960 data: 4 3 4 4 4 4 4 4 3 3 02/30 Read block 403 len 40960 03/30 Write block 541 len 16384 data: 2 2 3 4 04/30 Read block 339 len 8192 05/30 Write block 556 len 36864 data: 2 2 3 2 2 2 2 2 2 06/30 Read block 594 len 12288 07/30 Write block 179 len 24576 data: 2 2 2 3 3 3 08/30 Read block 238 len 20480 09/30 Write block 280 len 16384 data: 2 2 3 4 10/30 Read block 361 len 20480 11/30 Write block 421 len 8192 data: 2 2 12/30 Read block 40 len 40960 13/30 Read block 573 len 8192 14/30 Read block 234 len 40960 15/30 Read block 135 len 36864 16/30 Read block 502 len 20480 17/30 Read block 539 len 24576 18/30 Write block 330 len 8192 data: 2 2 19/30 Flush 20/30 Write block 590 len 4096 data: 2 21/30 Read block 504 len 12288 22/30 Write block 293 len 32768 data: 4 5 5 5 4 3 3 2 23/30 Read block 209 len 8192 24/30 Write block 101 len 20480 data: 4 5 4 4 4 25/30 Flush 26/30 Write block 194 len 12288 data: 2 2 2 27/30 Write block 403 len 4096 data: 3 28/30 Read block 516 len 20480 29/30 Write block 476 len 40960 data: 3 2 2 2 2 2 2 3 2 2 30/30 Write block 338 len 28672 data: 4 4 3 4 3 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 2 30 Done 4 28 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:30.039987112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1253} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 10 12 12 21 23 23 T F F <--- 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 11 12 12 22 24 24 F F F <--- 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 12 12 5 23 23 F F F <--- 9 180-199 8 12 12 16 23 23 F T T <--- 10 200-219 9 9 9 19 19 19 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 9 9 9 19 19 19 F F F 14 280-299 11 12 12 22 24 24 F F F <--- 15 300-319 1 12 12 1 24 24 F F F <--- 16 320-339 8 12 12 18 23 23 F T T <--- 17 340-359 8 8 8 17 17 17 F T T <--- 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 2 2 2 4 4 4 F T T <--- 21 420-439 10 12 12 20 23 23 F F F <--- 22 440-459 8 8 8 15 15 15 F F F 23 460-479 10 10 10 20 20 20 F T T <--- 24 480-499 10 10 10 21 21 21 F T T <--- 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 10 12 12 21 23 23 T F F <--- 28 560-579 2 12 12 6 23 23 F F F <--- 29 580-599 11 12 12 22 24 24 F F F <--- Max gen: 12, Max flush: 24 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 12 12 12 24 24 24 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 12 12 12 23 23 23 F F F 9 180-199 13 13 13 25 25 25 F F F 10 200-219 9 9 9 19 19 19 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 9 9 9 19 19 19 F F F 14 280-299 12 12 12 24 24 24 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 13 13 13 25 25 25 F F F 18 360-379 4 4 4 9 9 9 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 13 13 13 25 25 25 F F F 21 420-439 12 12 12 23 23 23 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 13 13 13 25 25 25 F F F 24 480-499 13 13 13 25 25 25 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 12 12 12 23 23 23 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 13, Max flush: 25 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.37040654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:44.38107823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.381123984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.391371727Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.391652631Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.391688191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.391700085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39171785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"Crucible ec97eaf8-a2a0-4861-859f-9d0f86e057ea has session id: 055693c0-5fa2-4eb5-afc7-00554334494a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392083989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Upstairs opts: Upstairs UUID: ec97eaf8-a2a0-4861-859f-9d0f86e057ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39211923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Crucible stats registered with UUID: ec97eaf8-a2a0-4861-859f-9d0f86e057ea","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392134784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:44.392148248Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.3922882Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392315168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392333013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392475166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39249047Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"ec97eaf8-a2a0-4861-859f-9d0f86e057ea active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392504084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392516938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392531312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392544987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392558631Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392603304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392626141Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63615)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392641855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39537)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.3926569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46058)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.392672054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39346362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393492739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393509664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393920346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[0]R flush_numbers[0..12]: [13, 23, 4, 16, 22, 24, 22, 1, 23, 25, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393946924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[0]R generation[0..12]: [7, 12, 2, 8, 11, 12, 11, 1, 12, 13, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393962839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393979764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[1]R flush_numbers[0..12]: [13, 23, 4, 16, 22, 24, 22, 1, 23, 25, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.393995489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[1]R generation[0..12]: [7, 12, 2, 8, 11, 12, 11, 1, 12, 13, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394011003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394027208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[2]R flush_numbers[0..12]: [13, 23, 4, 16, 22, 24, 22, 1, 23, 25, 19, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394042633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[2]R generation[0..12]: [7, 12, 2, 8, 11, 12, 11, 1, 12, 13, 9, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394057507Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394073952Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394087376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39410152Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394115074Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39413417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"":"downstairs","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394150205Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394162769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"0","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394177313Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"1","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394191237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"client":"2","":"downstairs","session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394205541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394219015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"ec97eaf8-a2a0-4861-859f-9d0f86e057ea is now active with session: 055693c0-5fa2-4eb5-afc7-00554334494a","v":0,"name":"crucible","level":30,"time":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2026-03-17T14:13:44.394251165Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.39426696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262,"session_id":"055693c0-5fa2-4eb5-afc7-00554334494a"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394281214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} {"msg":"Activated sub_volume ec97eaf8-a2a0-4861-859f-9d0f86e057ea","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:44.394292947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 355 len 32768 data: 3 3 3 2 2 2 2 2 03/30 Flush 04/30 Write block 41 len 24576 data: 2 2 2 2 2 2 05/30 Write block 342 len 36864 data: 4 4 4 3 4 4 4 4 5 06/30 Read block 244 len 8192 07/30 Read block 341 len 16384 08/30 Read block 96 len 32768 09/30 Write block 196 len 16384 data: 3 3 3 3 10/30 Read block 416 len 12288 11/30 Read block 64 len 40960 12/30 Read block 140 len 32768 13/30 Read block 286 len 4096 14/30 Read block 55 len 40960 15/30 Write block 267 len 8192 data: 4 4 16/30 Write block 561 len 16384 data: 3 3 3 3 17/30 Write block 191 len 40960 data: 3 3 3 3 3 4 4 4 4 3 18/30 Read block 93 len 32768 19/30 Read block 96 len 36864 20/30 Write block 353 len 12288 data: 3 3 4 21/30 Read block 422 len 36864 22/30 Write block 117 len 4096 data: 2 23/30 Read block 199 len 16384 24/30 Write block 399 len 40960 data: 3 3 3 3 4 2 2 2 2 2 25/30 Read block 392 len 28672 26/30 Write block 411 len 16384 data: 2 2 2 2 27/30 Read block 440 len 20480 28/30 Write block 78 len 36864 data: 4 4 3 3 3 3 2 2 2 29/30 Write block 350 len 12288 data: 6 3 3 30/30 Write block 431 len 24576 data: 2 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 3 3 35 Done 0 26 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:44.406359493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1262} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 2 2 2 4 4 4 F T T <--- 3 060-079 8 8 8 16 16 16 F T T <--- 4 080-099 11 11 11 22 22 22 F T T <--- 5 100-119 12 12 12 24 24 24 F T T <--- 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 12 12 12 23 23 23 F F F 9 180-199 13 13 13 25 25 25 F T T <--- 10 200-219 9 9 9 19 19 19 F T T <--- 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 9 9 9 19 19 19 F T T <--- 14 280-299 12 12 12 24 24 24 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 13 14 14 25 27 27 F T T <--- 18 360-379 4 14 14 9 27 27 F F F <--- 19 380-399 2 2 2 4 4 4 F T T <--- 20 400-419 13 13 13 25 25 25 F T T <--- 21 420-439 12 12 12 23 23 23 F T T <--- 22 440-459 8 8 8 15 15 15 F F F 23 460-479 13 13 13 25 25 25 F F F 24 480-499 13 13 13 25 25 25 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 12 12 12 23 23 23 F T T <--- 29 580-599 12 12 12 24 24 24 F F F Max gen: 14, Max flush: 27 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 15 15 15 28 28 28 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 12 12 12 23 23 23 F F F 9 180-199 15 15 15 28 28 28 F F F 10 200-219 15 15 15 28 28 28 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 15 15 15 28 28 28 F F F 14 280-299 12 12 12 24 24 24 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 14 14 14 27 27 27 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 13 13 13 25 25 25 F F F 24 480-499 13 13 13 25 25 25 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 28 28 28 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.735702431Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:58.746412562Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.746467468Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75674299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757018462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757055883Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757068347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757090363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"Crucible 230c0364-77df-4a64-8873-43bb5abc0f4c has session id: 239634b0-5bd2-4680-865b-0d93c6626dd3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757458063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Upstairs opts: Upstairs UUID: 230c0364-77df-4a64-8873-43bb5abc0f4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757495004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Crucible stats registered with UUID: 230c0364-77df-4a64-8873-43bb5abc0f4c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757511249Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:58.757524933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757667996Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757701706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75771724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757846389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757861763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"230c0364-77df-4a64-8873-43bb5abc0f4c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757877678Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757890362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757904276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.7579184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757932815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75795057Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757964984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63346)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75798374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50446)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.757998704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58080)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.758136925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.758838995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.758867973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.758921309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759290739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[0]R flush_numbers[0..12]: [13, 23, 28, 28, 28, 28, 22, 1, 23, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759314756Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[0]R generation[0..12]: [7, 12, 15, 15, 15, 15, 11, 1, 12, 15, 15, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759330511Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759347046Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[1]R flush_numbers[0..12]: [13, 23, 28, 28, 28, 28, 22, 1, 23, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75936247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[1]R generation[0..12]: [7, 12, 15, 15, 15, 15, 11, 1, 12, 15, 15, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759377855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75939437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[2]R flush_numbers[0..12]: [13, 23, 28, 28, 28, 28, 22, 1, 23, 28, 28, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759412695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[2]R generation[0..12]: [7, 12, 15, 15, 15, 15, 11, 1, 12, 15, 15, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75942808Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759445305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759458999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2026-03-17T14:13:58.759479285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75949357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759507164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"":"downstairs","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759521498Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759535092Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"0","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759549286Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"1","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.75956334Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"client":"2","":"downstairs","session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759578975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759592419Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"230c0364-77df-4a64-8873-43bb5abc0f4c is now active with session: 239634b0-5bd2-4680-865b-0d93c6626dd3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759607764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759622648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271,"session_id":"239634b0-5bd2-4680-865b-0d93c6626dd3"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759636072Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} {"msg":"Activated sub_volume 230c0364-77df-4a64-8873-43bb5abc0f4c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:13:58.759647415Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 32768 data: 3 3 2 2 3 3 3 4 02/30 Read block 84 len 8192 03/30 Read block 194 len 32768 04/30 Write block 191 len 4096 data: 4 05/30 Flush 06/30 Flush 07/30 Write block 421 len 28672 data: 3 3 2 2 2 2 2 08/30 Write block 462 len 8192 data: 2 2 09/30 Write block 75 len 16384 data: 3 3 3 5 10/30 Flush 11/30 Write block 467 len 36864 data: 4 3 4 4 3 3 3 3 3 12/30 Write block 21 len 4096 data: 2 13/30 Write block 196 len 36864 data: 5 5 5 5 4 3 3 4 4 14/30 Write block 373 len 28672 data: 2 2 2 2 2 3 3 15/30 Read block 45 len 16384 16/30 Read block 484 len 12288 17/30 Write block 285 len 32768 data: 5 4 3 3 4 4 4 5 18/30 Read block 70 len 40960 19/30 Write block 562 len 20480 data: 4 4 4 2 2 20/30 Write block 361 len 12288 data: 3 3 2 21/30 Write block 286 len 12288 data: 5 4 4 22/30 Read block 285 len 24576 23/30 Write block 460 len 28672 data: 2 2 3 3 2 3 4 24/30 Write block 267 len 28672 data: 5 5 4 4 5 5 5 25/30 Write block 435 len 12288 data: 4 4 3 26/30 Read block 357 len 20480 27/30 Read block 374 len 16384 28/30 Write block 101 len 40960 data: 5 6 5 5 5 3 3 3 3 3 29/30 Write block 22 len 24576 data: 2 2 2 2 2 2 30/30 Read block 2 len 24576 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:13:58.771305369Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1271} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 12 12 12 23 23 23 F T T <--- 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 16 16 28 31 31 F F F <--- 4 080-099 15 15 15 28 28 28 F F F 5 100-119 15 15 15 28 28 28 F T T <--- 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 12 16 16 23 29 29 T F F <--- 9 180-199 15 16 16 28 29 29 T T T <--- 10 200-219 15 15 15 28 28 28 F T T <--- 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 15 15 15 28 28 28 F T T <--- 14 280-299 12 12 12 24 24 24 F T T <--- 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 14 14 14 27 27 27 F T T <--- 19 380-399 15 15 15 28 28 28 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 16 16 28 31 31 F T T <--- 22 440-459 8 8 8 15 15 15 F F F 23 460-479 13 16 16 25 31 31 F T T <--- 24 480-499 13 13 13 25 25 25 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 15 15 15 28 28 28 F T T <--- 29 580-599 12 12 12 24 24 24 F F F Max gen: 16, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 16 16 16 31 31 31 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 17 17 17 32 32 32 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 17 17 17 32 32 32 F F F 10 200-219 17 17 17 32 32 32 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 F F F 13 260-279 17 17 17 32 32 32 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 17 17 17 32 32 32 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 17 17 17 32 32 32 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 17 17 17 32 32 32 F F F 24 480-499 13 13 13 25 25 25 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 F F F 28 560-579 17 17 17 32 32 32 F F F 29 580-599 12 12 12 24 24 24 F F F Max gen: 17, Max flush: 32 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.09699965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.108715911Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109026444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109064145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109076449Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109094524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"Crucible 4f5264f7-c51d-4362-8269-e8160ea208da has session id: 6f4ee76a-9f32-4eb2-8a63-e410ef9781d9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109433075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Upstairs opts: Upstairs UUID: 4f5264f7-c51d-4362-8269-e8160ea208da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109468186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Crucible stats registered with UUID: 4f5264f7-c51d-4362-8269-e8160ea208da","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.10948428Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:13.109497914Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109661513Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109697574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109716279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109842597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109857522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"4f5264f7-c51d-4362-8269-e8160ea208da active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109873636Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.1098864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109900534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109914729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109928523Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109942957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.109957471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38490)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110022981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63664)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110191341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46606)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110219499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110789049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110818158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.110875445Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111206153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[0]R flush_numbers[0..12]: [13, 32, 28, 31, 28, 32, 22, 1, 29, 32, 32, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.1112282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[0]R generation[0..12]: [7, 17, 15, 16, 15, 17, 11, 1, 16, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111244065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.11126117Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[1]R flush_numbers[0..12]: [13, 32, 28, 31, 28, 32, 22, 1, 29, 32, 32, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111276684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[1]R generation[0..12]: [7, 17, 15, 16, 15, 17, 11, 1, 16, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111292259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111308514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[2]R flush_numbers[0..12]: [13, 32, 28, 31, 28, 32, 22, 1, 29, 32, 32, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111323848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[2]R generation[0..12]: [7, 17, 15, 16, 15, 17, 11, 1, 16, 17, 17, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111338763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111355038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111368732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111382806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111401232Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111415426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"":"downstairs","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111431681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111444494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"0","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111458809Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"1","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111472633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"client":"2","":"downstairs","session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111486987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:14:13.111500531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"4f5264f7-c51d-4362-8269-e8160ea208da is now active with session: 6f4ee76a-9f32-4eb2-8a63-e410ef9781d9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111518336Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.11153118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280,"session_id":"6f4ee76a-9f32-4eb2-8a63-e410ef9781d9"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111544584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} {"msg":"Activated sub_volume 4f5264f7-c51d-4362-8269-e8160ea208da","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:13.111556058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 16384 data: 2 2 3 3 02/30 Read block 250 len 20480 03/30 Read block 595 len 4096 04/30 Read block 499 len 20480 05/30 Read block 82 len 40960 06/30 Read block 409 len 20480 07/30 Read block 227 len 32768 08/30 Read block 74 len 4096 09/30 Read block 190 len 20480 10/30 Read block 287 len 4096 11/30 Write block 385 len 28672 data: 2 2 2 2 2 2 2 12/30 Read block 307 len 8192 13/30 Read block 189 len 8192 14/30 Read block 268 len 40960 15/30 Read block 254 len 20480 16/30 Read block 370 len 20480 17/30 Write block 43 len 24576 data: 3 3 3 3 2 2 18/30 Read block 332 len 24576 19/30 Write block 574 len 32768 data: 3 3 3 2 2 2 3 3 20/30 Read block 453 len 28672 21/30 Write block 209 len 36864 data: 4 2 2 2 2 2 2 2 2 22/30 Read block 146 len 40960 23/30 Write block 488 len 12288 data: 2 2 2 24/30 Write block 76 len 36864 data: 4 4 6 5 4 4 4 4 3 25/30 Write block 255 len 24576 data: 2 2 3 3 3 3 26/30 Write block 549 len 28672 data: 4 4 4 3 2 2 2 27/30 Write block 204 len 32768 data: 5 3 3 3 4 5 3 3 28/30 Read block 495 len 16384 29/30 Write block 505 len 24576 data: 3 3 3 3 3 3 30/30 Read block 502 len 36864 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 2 31 Done 30 1 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:13.123720982Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 15 15 15 28 28 28 T F T <--- 3 060-079 16 16 16 31 31 31 T F T <--- 4 080-099 15 15 15 28 28 28 T F T <--- 5 100-119 17 17 17 32 32 32 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 17 17 17 32 32 32 F F F 10 200-219 17 17 17 32 32 32 T F T <--- 11 220-239 10 10 10 20 20 20 F F F 12 240-259 10 10 10 20 20 20 T F T <--- 13 260-279 17 17 17 32 32 32 T F T <--- 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 17 17 17 32 32 32 T T T 19 380-399 15 15 15 28 28 28 T F T <--- 20 400-419 15 15 15 28 28 28 F F F 21 420-439 17 17 17 32 32 32 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 17 17 17 32 32 32 F F F 24 480-499 13 13 13 25 25 25 T F T <--- 25 500-519 9 9 9 19 19 19 T F T <--- 26 520-539 6 6 6 11 11 11 F F F 27 540-559 12 12 12 23 23 23 T F T <--- 28 560-579 17 17 17 32 32 32 T F T <--- 29 580-599 12 12 12 24 24 24 T F T <--- Max gen: 17, Max flush: 32 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 18 18 18 33 33 33 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 18 18 18 33 33 33 F F F 5 100-119 17 17 17 32 32 32 F F F 6 120-139 11 11 11 22 22 22 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 17 17 17 32 32 32 F F F 10 200-219 18 18 18 33 33 33 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 18 18 18 33 33 33 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 18 18 18 33 33 33 F F F 19 380-399 18 18 18 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 17 17 17 32 32 32 F F F 22 440-459 8 8 8 15 15 15 F F F 23 460-479 17 17 17 32 32 32 F F F 24 480-499 18 18 18 33 33 33 F F F 25 500-519 18 18 18 33 33 33 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 18 18 18 33 33 33 F F F 28 560-579 18 18 18 33 33 33 F F F 29 580-599 18 18 18 33 33 33 F F F Max gen: 18, Max flush: 33 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.454648344Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:27.465354934Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.465406589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.475885322Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476186872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476227254Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476239257Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476260894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"Crucible 7ae01e8a-1819-4b01-a34e-6165f534f269 has session id: 84f24c14-514b-46ba-ab4f-29c1d6c5f129","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476580469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Upstairs opts: Upstairs UUID: 7ae01e8a-1819-4b01-a34e-6165f534f269, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476606567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Crucible stats registered with UUID: 7ae01e8a-1819-4b01-a34e-6165f534f269","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476621491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:27.476634895Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476796673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.4768208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476836565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476974266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.476991761Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"7ae01e8a-1819-4b01-a34e-6165f534f269 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477013498Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477026322Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477040176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.4770538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477067554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477110597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477134304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"ds_connection connected from Ok(127.0.0.1:37674)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477164183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"ds_connection connected from Ok(127.0.0.1:50363)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477179697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"ds_connection connected from Ok(127.0.0.1:36866)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477194652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.477998841Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.47802697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478043765Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478477784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[0]R flush_numbers[0..12]: [13, 32, 33, 33, 33, 32, 22, 1, 29, 32, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478499951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[0]R generation[0..12]: [7, 17, 18, 18, 18, 17, 11, 1, 16, 17, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478515815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.4785325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[1]R flush_numbers[0..12]: [13, 32, 33, 33, 33, 32, 22, 1, 29, 32, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478548065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[1]R generation[0..12]: [7, 17, 18, 18, 18, 17, 11, 1, 16, 17, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.47856326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478583776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[2]R flush_numbers[0..12]: [13, 32, 33, 33, 33, 32, 22, 1, 29, 32, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.47859963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[2]R generation[0..12]: [7, 17, 18, 18, 18, 17, 11, 1, 16, 17, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478615015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.47863173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Max found gen is 19","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478644994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Generation requested: 20 >= found:19","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:14:27.478659158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478677174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478690417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"":"downstairs","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478704712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478717396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"0","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.47873146Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"1","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478745774Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"client":"2","":"downstairs","session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478760168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478776633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"7ae01e8a-1819-4b01-a34e-6165f534f269 is now active with session: 84f24c14-514b-46ba-ab4f-29c1d6c5f129","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478791858Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478806402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289,"session_id":"84f24c14-514b-46ba-ab4f-29c1d6c5f129"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478820016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} {"msg":"Activated sub_volume 7ae01e8a-1819-4b01-a34e-6165f534f269","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:27.478831319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 32768 data: 3 3 4 3 3 4 5 5 02/30 Read block 146 len 12288 03/30 Flush 04/30 Flush 05/30 Write block 458 len 24576 data: 2 2 3 3 4 4 06/30 Read block 487 len 12288 07/30 Read block 120 len 12288 08/30 Write block 547 len 8192 data: 5 5 09/30 Read block 134 len 8192 10/30 Write block 530 len 12288 data: 3 3 3 11/30 Write block 385 len 4096 data: 3 12/30 Write block 341 len 36864 data: 5 5 5 5 4 5 5 5 5 13/30 Read block 444 len 40960 14/30 Write block 16 len 16384 data: 2 2 3 3 15/30 Flush 16/30 Flush 17/30 Write block 182 len 32768 data: 5 4 4 4 3 3 4 3 18/30 Write block 147 len 20480 data: 2 2 2 2 2 19/30 Read block 45 len 36864 20/30 Flush 21/30 Flush 22/30 Write block 475 len 32768 data: 4 4 3 3 3 3 3 3 23/30 Read block 14 len 8192 24/30 Write block 577 len 28672 data: 3 3 3 4 4 3 3 25/30 Read block 94 len 40960 26/30 Write block 455 len 12288 data: 2 2 2 27/30 Read block 215 len 4096 28/30 Read block 297 len 8192 29/30 Write block 128 len 40960 data: 4 3 3 3 3 3 3 3 2 2 30/30 Write block 414 len 12288 data: 3 2 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:27.49138457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1289} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 20 20 13 36 36 F F F <--- 1 020-039 17 17 17 32 32 32 F F F 2 040-059 18 18 18 33 33 33 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 18 18 18 33 33 33 F F F 5 100-119 17 17 17 32 32 32 F F F 6 120-139 11 11 11 22 22 22 F T T <--- 7 140-159 1 20 20 1 38 38 F F F <--- 8 160-179 16 16 16 29 29 29 F F F 9 180-199 17 20 20 32 38 38 F F F <--- 10 200-219 18 18 18 33 33 33 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 18 18 18 33 33 33 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 15 20 20 28 36 36 F F F <--- 18 360-379 18 18 18 33 33 33 F F F 19 380-399 18 20 20 33 36 36 F F F <--- 20 400-419 15 15 15 28 28 28 F T T <--- 21 420-439 17 17 17 32 32 32 F F F 22 440-459 8 20 20 15 36 36 F T T <--- 23 460-479 17 20 20 32 36 36 F T T <--- 24 480-499 18 18 18 33 33 33 F T T <--- 25 500-519 18 18 18 33 33 33 F F F 26 520-539 6 20 20 11 36 36 F F F <--- 27 540-559 18 20 20 33 36 36 F F F <--- 28 560-579 18 20 20 33 34 34 F T T <--- 29 580-599 18 18 18 33 33 33 F T T <--- Max gen: 20, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 18 18 18 33 33 33 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 18 18 18 33 33 33 F F F 5 100-119 17 17 17 32 32 32 F F F 6 120-139 21 21 21 39 39 39 F F F 7 140-159 20 20 20 38 38 38 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 20 20 20 38 38 38 F F F 10 200-219 18 18 18 33 33 33 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 18 18 18 33 33 33 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 20 20 20 36 36 36 F F F 18 360-379 18 18 18 33 33 33 F F F 19 380-399 20 20 20 36 36 36 F F F 20 400-419 21 21 21 39 39 39 F F F 21 420-439 17 17 17 32 32 32 F F F 22 440-459 21 21 21 39 39 39 F F F 23 460-479 21 21 21 39 39 39 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 18 18 18 33 33 33 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 36 36 36 F F F 28 560-579 21 21 21 39 39 39 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.821056648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.832802518Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833116531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833154813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833166796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833184852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"Crucible a28e9309-9fc1-4934-a784-42f62729eb8a has session id: a3ed1bb8-27d8-48f4-bae6-4659090b4b2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833541448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Upstairs opts: Upstairs UUID: a28e9309-9fc1-4934-a784-42f62729eb8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833566915Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Crucible stats registered with UUID: a28e9309-9fc1-4934-a784-42f62729eb8a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.83358174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:41.833595314Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833771206Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833806077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833821241Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833957092Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833972386Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"a28e9309-9fc1-4934-a784-42f62729eb8a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.833989851Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834002715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834016909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834030694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834044478Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834058732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834082739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38924)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834208126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54079)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834333484Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60276)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834361872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834908585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.834948527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835165242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835550056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[0]R flush_numbers[0..12]: [36, 32, 33, 33, 33, 32, 39, 38, 29, 38, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835571763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[0]R generation[0..12]: [20, 17, 18, 18, 18, 17, 21, 20, 16, 20, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835587497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835603852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[1]R flush_numbers[0..12]: [36, 32, 33, 33, 33, 32, 39, 38, 29, 38, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835619397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[1]R generation[0..12]: [20, 17, 18, 18, 18, 17, 21, 20, 16, 20, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835635022Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835651707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[2]R flush_numbers[0..12]: [36, 32, 33, 33, 33, 32, 39, 38, 29, 38, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835667051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[2]R generation[0..12]: [20, 17, 18, 18, 18, 17, 21, 20, 16, 20, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835682106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.83569786Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835711414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Generation requested: 22 >= found:22",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.83573163Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835744894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835758608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"":"downstairs","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835772853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835788067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"0","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835802091Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"1","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835816056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"client":"2","":"downstairs","session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.83583048Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:14:41.835844054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"a28e9309-9fc1-4934-a784-42f62729eb8a is now active with session: a3ed1bb8-27d8-48f4-bae6-4659090b4b2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835861309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835874133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298,"session_id":"a3ed1bb8-27d8-48f4-bae6-4659090b4b2e"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.835887547Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} {"msg":"Activated sub_volume a28e9309-9fc1-4934-a784-42f62729eb8a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:41.83589891Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 12288 data: 4 3 3 02/30 Read block 338 len 36864 03/30 Write block 132 len 36864 data: 4 4 4 4 3 3 2 2 2 04/30 Write block 251 len 8192 data: 3 2 05/30 Flush 06/30 Write block 562 len 12288 data: 6 6 5 07/30 Flush 08/30 Write block 10 len 32768 data: 2 2 2 2 3 3 3 3 09/30 Write block 190 len 36864 data: 3 5 4 4 4 4 6 6 6 10/30 Read block 31 len 28672 11/30 Read block 172 len 4096 12/30 Read block 433 len 28672 13/30 Read block 41 len 36864 14/30 Write block 304 len 28672 data: 2 2 2 2 2 2 2 15/30 Read block 565 len 28672 16/30 Write block 47 len 32768 data: 3 3 2 2 2 2 2 2 17/30 Write block 93 len 32768 data: 3 3 3 4 4 4 4 4 18/30 Write block 427 len 12288 data: 3 2 2 19/30 Read block 225 len 28672 20/30 Read block 566 len 20480 21/30 Read block 133 len 28672 22/30 Read block 434 len 36864 23/30 Write block 137 len 8192 data: 4 3 24/30 Read block 104 len 32768 25/30 Write block 308 len 8192 data: 3 3 26/30 Write block 149 len 20480 data: 3 3 3 2 2 27/30 Read block 341 len 36864 28/30 Write block 568 len 24576 data: 2 3 3 3 3 3 29/30 Write block 172 len 12288 data: 2 3 3 30/30 Write block 171 len 16384 data: 2 3 4 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 8 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 18 22 Done 21 21 5 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:41.847756203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1298} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 T T T 1 020-039 17 17 17 32 32 32 F F F 2 040-059 18 18 18 33 33 33 T T F <--- 3 060-079 18 18 18 33 33 33 F F F 4 080-099 18 18 18 33 33 33 T T F <--- 5 100-119 17 17 17 32 32 32 T T F <--- 6 120-139 22 22 22 40 40 40 T T F <--- 7 140-159 22 22 22 40 40 40 T T F <--- 8 160-179 16 16 16 29 29 29 T T F <--- 9 180-199 20 20 20 38 38 38 T T T 10 200-219 18 18 18 33 33 33 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 12 12 12 24 24 24 T T F <--- 16 320-339 13 13 13 25 25 25 F F F 17 340-359 20 20 20 36 36 36 F F F 18 360-379 18 18 18 33 33 33 F F F 19 380-399 22 22 22 40 40 40 F F F 20 400-419 21 21 21 39 39 39 F F F 21 420-439 17 17 17 32 32 32 T T F <--- 22 440-459 21 21 21 39 39 39 F F F 23 460-479 21 21 21 39 39 39 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 18 18 18 33 33 33 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 36 36 36 F F F 28 560-579 22 22 22 41 41 41 T T F <--- 29 580-599 21 21 21 39 39 39 F F F Max gen: 22, Max flush: 41 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 42 42 42 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 23 23 23 42 42 42 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 23 23 23 42 42 42 F F F 8 160-179 23 23 23 42 42 42 F F F 9 180-199 23 23 23 42 42 42 F F F 10 200-219 18 18 18 33 33 33 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 17 17 32 32 32 F F F 15 300-319 23 23 23 42 42 42 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 20 20 20 36 36 36 F F F 18 360-379 18 18 18 33 33 33 F F F 19 380-399 22 22 22 40 40 40 F F F 20 400-419 21 21 21 39 39 39 F F F 21 420-439 23 23 23 42 42 42 F F F 22 440-459 21 21 21 39 39 39 F F F 23 460-479 21 21 21 39 39 39 F F F 24 480-499 21 21 21 39 39 39 F F F 25 500-519 18 18 18 33 33 33 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 36 36 36 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 21 21 21 39 39 39 F F F Max gen: 23, Max flush: 42 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.169418843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:56.180198005Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.180245759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.190607056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.190892051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.190932183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.190944297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.190962032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"Crucible 55bbc5b5-04bb-4f74-876b-5a4c098b885a has session id: 4c04baed-0c6c-474d-a439-5b4e3288b5f9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191322149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Upstairs opts: Upstairs UUID: 55bbc5b5-04bb-4f74-876b-5a4c098b885a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.19135632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Crucible stats registered with UUID: 55bbc5b5-04bb-4f74-876b-5a4c098b885a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191371944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:56.191385358Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.19152573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191550587Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191565532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.19172551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191741144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"55bbc5b5-04bb-4f74-876b-5a4c098b885a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191771503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191785477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191799872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191813936Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.19182755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191841884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191856219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46529)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191874264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58794)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191889558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63010)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.191976364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.192674442Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.192702711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.192777123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193119815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[0]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 42, 42, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193143362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[0]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 23, 23, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193159187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193175572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[1]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 42, 42, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193191286Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[1]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 23, 23, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193206871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193223446Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[2]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 42, 42, 33, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.19323888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[2]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 23, 23, 18, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193257306Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193273981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193287375Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:14:56.193301479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193319024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193332838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"":"downstairs","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193346843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193359666Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"0","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193373761Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"1","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193387645Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"client":"2","":"downstairs","session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193401719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193418424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"55bbc5b5-04bb-4f74-876b-5a4c098b885a is now active with session: 4c04baed-0c6c-474d-a439-5b4e3288b5f9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193434109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193447113Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307,"session_id":"4c04baed-0c6c-474d-a439-5b4e3288b5f9"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193460897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} {"msg":"Activated sub_volume 55bbc5b5-04bb-4f74-876b-5a4c098b885a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:14:56.193474191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 16384 data: 4 3 2 2 02/30 Read block 566 len 16384 03/30 Write block 343 len 8192 data: 6 6 04/30 Read block 472 len 16384 05/30 Read block 456 len 36864 06/30 Read block 12 len 28672 07/30 Write block 533 len 28672 data: 3 3 3 3 3 2 2 08/30 Write block 465 len 20480 data: 4 5 5 4 5 09/30 Read block 174 len 36864 10/30 Read block 4 len 40960 11/30 Read block 161 len 32768 12/30 Write block 179 len 36864 data: 4 4 4 6 5 5 5 4 4 13/30 Read block 525 len 40960 14/30 Write block 349 len 40960 data: 6 7 4 4 4 4 5 4 4 3 15/30 Write block 593 len 12288 data: 3 3 3 16/30 Read block 129 len 4096 17/30 Write block 201 len 8192 data: 4 4 18/30 Write block 536 len 24576 data: 4 4 3 3 2 3 19/30 Read block 569 len 8192 20/30 Write block 175 len 24576 data: 4 4 3 3 5 5 21/30 Read block 126 len 28672 22/30 Read block 553 len 4096 23/30 Flush 24/30 Write block 448 len 36864 data: 2 2 2 3 3 3 2 3 3 25/30 Read block 22 len 20480 26/30 Read block 222 len 16384 27/30 Write block 441 len 20480 data: 2 2 3 3 3 28/30 Read block 128 len 8192 29/30 Read block 220 len 28672 30/30 Write block 478 len 16384 data: 4 4 4 4 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:14:56.206636052Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1307} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 42 42 42 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 23 23 23 42 42 42 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 23 23 23 42 42 42 F F F 8 160-179 23 24 24 42 43 43 F F F <--- 9 180-199 23 24 24 42 43 43 F F F <--- 10 200-219 18 24 24 33 43 43 F F F <--- 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 17 24 24 32 43 43 T F F <--- 15 300-319 23 24 24 42 43 43 T F F <--- 16 320-339 13 13 13 25 25 25 F F F 17 340-359 20 24 24 36 43 43 T F F <--- 18 360-379 18 18 18 33 33 33 F F F 19 380-399 22 22 22 40 40 40 F F F 20 400-419 21 21 21 39 39 39 F F F 21 420-439 23 23 23 42 42 42 F F F 22 440-459 21 21 21 39 39 39 F T T <--- 23 460-479 21 24 24 39 43 43 F T T <--- 24 480-499 21 21 21 39 39 39 F T T <--- 25 500-519 18 18 18 33 33 33 F F F 26 520-539 20 24 24 36 43 43 F F F <--- 27 540-559 20 24 24 36 43 43 F F F <--- 28 560-579 23 23 23 42 42 42 F F F 29 580-599 21 24 24 39 43 43 F F F <--- Max gen: 24, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 42 42 42 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 23 23 23 42 42 42 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 23 23 23 42 42 42 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 23 23 23 42 42 42 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 24 24 24 43 43 43 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 24 24 24 43 43 43 F F F 15 300-319 24 24 24 43 43 43 F F F 16 320-339 13 13 13 25 25 25 F F F 17 340-359 24 24 24 43 43 43 F F F 18 360-379 18 18 18 33 33 33 F F F 19 380-399 22 22 22 40 40 40 F F F 20 400-419 21 21 21 39 39 39 F F F 21 420-439 23 23 23 42 42 42 F F F 22 440-459 25 25 25 44 44 44 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 25 25 25 44 44 44 F F F 25 500-519 18 18 18 33 33 33 F F F 26 520-539 24 24 24 43 43 43 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 24 24 24 43 43 43 F F F Max gen: 25, Max flush: 44 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.531760924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.543521649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.543838363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.543876824Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.543889178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.543906983Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"Crucible 281e3045-7338-42c1-92d2-81c6a041abd4 has session id: fdae101d-cd7e-4dde-a551-5a85c5f10bdd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.54426464Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Upstairs opts: Upstairs UUID: 281e3045-7338-42c1-92d2-81c6a041abd4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544310444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Crucible stats registered with UUID: 281e3045-7338-42c1-92d2-81c6a041abd4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544327129Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:10.544340373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.54450039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544535291Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544551145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544688496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544704381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"281e3045-7338-42c1-92d2-81c6a041abd4 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544722326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.54473542Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544749584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544763719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544777283Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544791747Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544827278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44556)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544850735Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58136)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544866139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57863)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.544881084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.545509591Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.545554874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546209469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546532466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[0]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 43, 43, 43, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546556203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[0]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 24, 24, 24, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546575178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546591803Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[1]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 43, 43, 43, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546607488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[1]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 24, 24, 24, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546622983Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546639328Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[2]R flush_numbers[0..12]: [42, 32, 42, 33, 42, 42, 42, 42, 43, 43, 43, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546654632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[2]R generation[0..12]: [23, 17, 23, 18, 23, 23, 23, 23, 24, 24, 24, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546669807Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546686091Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546699475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:15:10.54671361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546733065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546746639Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"":"downstairs","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546760434Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546773037Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"0","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546789152Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"1","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546803276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"client":"2","":"downstairs","session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546817831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:15:10.546831425Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"281e3045-7338-42c1-92d2-81c6a041abd4 is now active with session: fdae101d-cd7e-4dde-a551-5a85c5f10bdd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.54684951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546862164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316,"session_id":"fdae101d-cd7e-4dde-a551-5a85c5f10bdd"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546875448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} {"msg":"Activated sub_volume 281e3045-7338-42c1-92d2-81c6a041abd4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:10.546887271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 4096 data: 4 02/30 Write block 216 len 4096 data: 3 03/30 Write block 49 len 20480 data: 3 3 3 3 3 04/30 Read block 253 len 20480 05/30 Write block 583 len 36864 data: 4 2 2 3 3 3 2 3 3 06/30 Write block 513 len 36864 data: 3 3 2 2 3 3 2 2 2 07/30 Write block 503 len 40960 data: 3 3 4 4 4 4 4 4 3 3 08/30 Read block 217 len 4096 09/30 Read block 540 len 32768 10/30 Flush 11/30 Flush 12/30 Write block 402 len 16384 data: 4 5 3 3 13/30 Read block 329 len 24576 14/30 Read block 287 len 36864 15/30 Read block 4 len 40960 16/30 Write block 449 len 28672 data: 3 3 4 4 4 3 4 17/30 Write block 81 len 28672 data: 5 5 5 4 3 3 2 18/30 Read block 154 len 4096 19/30 Read block 481 len 8192 20/30 Write block 513 len 12288 data: 4 4 3 21/30 Read block 193 len 12288 22/30 Read block 41 len 28672 23/30 Write block 283 len 36864 data: 5 5 6 6 5 5 5 5 5 24/30 Read block 569 len 36864 25/30 Write block 315 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Write block 372 len 24576 data: 3 4 4 3 3 3 27/30 Write block 385 len 4096 data: 5 28/30 Write block 159 len 4096 data: 2 29/30 Read block 185 len 32768 30/30 Write block 349 len 40960 data: 7 8 5 5 5 5 6 5 5 4 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 29 29 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:10.55869593Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1316} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 23 45 45 42 F F T <--- 1 020-039 17 17 17 32 32 32 F F F 2 040-059 26 26 23 45 45 42 F F T <--- 3 060-079 18 18 18 33 33 33 F F F 4 080-099 23 23 23 42 42 42 T T F <--- 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 23 23 23 42 42 42 T T F <--- 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 24 45 45 43 F F T <--- 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 24 24 24 43 43 43 T T F <--- 15 300-319 24 24 24 43 43 43 T T F <--- 16 320-339 13 13 13 25 25 25 T T F <--- 17 340-359 24 24 24 43 43 43 T T F <--- 18 360-379 18 18 18 33 33 33 T T F <--- 19 380-399 22 22 22 40 40 40 T T F <--- 20 400-419 21 21 21 39 39 39 T T F <--- 21 420-439 23 23 23 42 42 42 F F F 22 440-459 25 25 25 44 44 44 T T F <--- 23 460-479 25 25 25 44 44 44 F F F 24 480-499 25 25 25 44 44 44 F F F 25 500-519 26 26 18 45 45 33 T T F <--- 26 520-539 26 26 24 45 45 43 F F F <--- 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 26 26 24 45 45 43 F F F <--- Max gen: 26, Max flush: 45 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 17 17 17 32 32 32 F F F 2 040-059 26 26 26 45 45 45 F F F 3 060-079 18 18 18 33 33 33 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 10 10 10 20 20 20 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 27 27 27 46 46 46 F F F 15 300-319 27 27 27 46 46 46 F F F 16 320-339 27 27 27 46 46 46 F F F 17 340-359 27 27 27 46 46 46 F F F 18 360-379 27 27 27 46 46 46 F F F 19 380-399 27 27 27 46 46 46 F F F 20 400-419 27 27 27 46 46 46 F F F 21 420-439 23 23 23 42 42 42 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 25 25 25 44 44 44 F F F 25 500-519 27 27 27 46 46 46 F F F 26 520-539 26 26 26 45 45 45 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 26 26 26 45 45 45 F F F Max gen: 27, Max flush: 46 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.896976612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:24.907700067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.907745131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.917985052Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918294594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918331725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918343809Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918361944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"Crucible 3f65b4b6-ab66-44c1-b80e-fb406dca1f1b has session id: df4f1d20-fbcc-4725-b5c1-e522bef98381","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918700955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Upstairs opts: Upstairs UUID: 3f65b4b6-ab66-44c1-b80e-fb406dca1f1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918736586Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Crucible stats registered with UUID: 3f65b4b6-ab66-44c1-b80e-fb406dca1f1b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918752401Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:24.918766175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918912688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918940867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.918955591Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919109967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919126862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"3f65b4b6-ab66-44c1-b80e-fb406dca1f1b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919149369Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919162193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919176247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919190291Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919203755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.91921859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919233164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"ds_connection connected from Ok(127.0.0.1:49843)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919251539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"ds_connection connected from Ok(127.0.0.1:64728)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919407106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"ds_connection connected from Ok(127.0.0.1:50361)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919434424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.919956429Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.91999223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920054439Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920418677Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[0]R flush_numbers[0..12]: [45, 32, 45, 33, 46, 42, 42, 46, 43, 43, 45, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920440254Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[0]R generation[0..12]: [26, 17, 26, 18, 27, 23, 23, 27, 24, 24, 26, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920456318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920473073Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[1]R flush_numbers[0..12]: [45, 32, 45, 33, 46, 42, 42, 46, 43, 43, 45, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920488688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[1]R generation[0..12]: [26, 17, 26, 18, 27, 23, 23, 27, 24, 24, 26, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920504023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920520407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[2]R flush_numbers[0..12]: [45, 32, 45, 33, 46, 42, 42, 46, 43, 43, 45, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920539173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[2]R generation[0..12]: [26, 17, 26, 18, 27, 23, 23, 27, 24, 24, 26, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920554878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920571483Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920585437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:15:24.920599811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920617967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920631391Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"":"downstairs","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920645145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920657868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"0","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920672543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"1","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920686787Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"client":"2","":"downstairs","session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920701211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920717536Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"3f65b4b6-ab66-44c1-b80e-fb406dca1f1b is now active with session: df4f1d20-fbcc-4725-b5c1-e522bef98381","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920733051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920745805Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325,"session_id":"df4f1d20-fbcc-4725-b5c1-e522bef98381"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920759069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} {"msg":"Activated sub_volume 3f65b4b6-ab66-44c1-b80e-fb406dca1f1b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:24.920772242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 20480 data: 4 4 3 4 4 02/30 Read block 272 len 4096 03/30 Read block 59 len 36864 04/30 Write block 329 len 24576 data: 2 3 3 2 2 4 05/30 Read block 343 len 12288 06/30 Write block 233 len 12288 data: 2 2 3 07/30 Read block 365 len 36864 08/30 Read block 30 len 12288 09/30 Read block 214 len 4096 10/30 Read block 546 len 4096 11/30 Write block 423 len 12288 data: 3 3 3 12/30 Flush 13/30 Write block 482 len 24576 data: 4 4 3 3 2 2 14/30 Write block 23 len 12288 data: 3 3 3 15/30 Read block 206 len 24576 16/30 Write block 68 len 4096 data: 4 17/30 Flush 18/30 Write block 333 len 16384 data: 3 5 4 4 19/30 Read block 261 len 40960 20/30 Read block 569 len 16384 21/30 Read block 448 len 4096 22/30 Flush 23/30 Write block 299 len 28672 data: 5 4 3 3 2 3 3 24/30 Read block 297 len 20480 25/30 Read block 129 len 40960 26/30 Read block 139 len 8192 27/30 Read block 358 len 28672 28/30 Write block 57 len 8192 data: 2 2 29/30 Write block 521 len 28672 data: 3 2 2 2 2 2 2 30/30 Read block 221 len 24576 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 1 26 Done 5 30 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:24.932978389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1325} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 17 28 28 32 48 48 F F F <--- 2 040-059 26 26 26 45 45 45 F T T <--- 3 060-079 18 28 28 33 48 48 F F F <--- 4 080-099 27 27 27 46 46 46 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 10 28 28 20 47 47 F F F <--- 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 27 27 27 46 46 46 F T T <--- 15 300-319 27 27 27 46 46 46 F T T <--- 16 320-339 27 28 28 46 49 49 T F F <--- 17 340-359 27 27 27 46 46 46 F F F 18 360-379 27 27 27 46 46 46 F F F 19 380-399 27 27 27 46 46 46 F F F 20 400-419 27 27 27 46 46 46 F F F 21 420-439 23 28 28 42 47 47 F F F <--- 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 25 28 28 44 48 48 F F F <--- 25 500-519 27 27 27 46 46 46 F F F 26 520-539 26 26 26 45 45 45 F T T <--- 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 26 28 28 45 47 47 T F F <--- Max gen: 28, Max flush: 49 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 29 29 29 50 50 50 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 23 23 23 42 42 42 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 F F F 8 160-179 24 24 24 43 43 43 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 28 28 28 49 49 49 F F F 17 340-359 27 27 27 46 46 46 F F F 18 360-379 27 27 27 46 46 46 F F F 19 380-399 27 27 27 46 46 46 F F F 20 400-419 27 27 27 46 46 46 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 28 28 28 48 48 48 F F F 25 500-519 27 27 27 46 46 46 F F F 26 520-539 29 29 29 50 50 50 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 28 28 28 47 47 47 F F F Max gen: 29, Max flush: 50 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.256758821Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:39.267457319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.267514016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.27783169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278134921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278172752Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278185196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278203141Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"Crucible 6cf0d5d7-5e38-498e-b407-296a6e8a0597 has session id: c6f74236-ae51-4770-91ad-18925e7af40e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278544653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Upstairs opts: Upstairs UUID: 6cf0d5d7-5e38-498e-b407-296a6e8a0597, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278584015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Crucible stats registered with UUID: 6cf0d5d7-5e38-498e-b407-296a6e8a0597","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278598679Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:39.278611943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278776582Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278811902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278827837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.278957086Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.27897157Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"6cf0d5d7-5e38-498e-b407-296a6e8a0597 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279012982Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279027347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279041711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279056015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279070059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279084994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279099478Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47998)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279213232Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55313)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279337989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38601)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279366738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279933967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.279961965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280013961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280398725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[0]R flush_numbers[0..12]: [45, 48, 50, 48, 46, 42, 42, 46, 43, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280422102Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[0]R generation[0..12]: [26, 28, 29, 28, 27, 23, 23, 27, 24, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280438007Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280454272Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[1]R flush_numbers[0..12]: [45, 48, 50, 48, 46, 42, 42, 46, 43, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280470106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[1]R generation[0..12]: [26, 28, 29, 28, 27, 23, 23, 27, 24, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280485131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280501646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[2]R flush_numbers[0..12]: [45, 48, 50, 48, 46, 42, 42, 46, 43, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.28051679Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[2]R generation[0..12]: [26, 28, 29, 28, 27, 23, 23, 27, 24, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280532215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.28054871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280562264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:15:39.280576338Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280595634Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280609098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"":"downstairs","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280622862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280638317Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"0","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280652191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"1","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280665645Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"client":"2","":"downstairs","session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280679889Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280696294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"6cf0d5d7-5e38-498e-b407-296a6e8a0597 is now active with session: c6f74236-ae51-4770-91ad-18925e7af40e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280711408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280724202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334,"session_id":"c6f74236-ae51-4770-91ad-18925e7af40e"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.280737596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} {"msg":"Activated sub_volume 6cf0d5d7-5e38-498e-b407-296a6e8a0597","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:39.28074898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 107 len 32768 data: 4 4 4 4 2 2 3 3 03/30 Read block 200 len 24576 04/30 Flush 05/30 Write block 342 len 20480 data: 6 7 7 5 6 06/30 Write block 47 len 36864 data: 4 4 4 4 4 4 4 3 2 07/30 Read block 101 len 40960 08/30 Read block 468 len 8192 09/30 Write block 380 len 8192 data: 2 2 10/30 Read block 21 len 32768 11/30 Read block 85 len 20480 12/30 Read block 528 len 8192 13/30 Read block 418 len 12288 14/30 Write block 398 len 20480 data: 3 4 4 4 5 15/30 Write block 325 len 8192 data: 2 2 16/30 Write block 174 len 16384 data: 5 5 5 4 17/30 Read block 393 len 16384 18/30 Read block 147 len 16384 19/30 Read block 253 len 12288 20/30 Write block 527 len 36864 data: 3 2 3 4 4 4 4 4 4 21/30 Flush 22/30 Write block 590 len 16384 data: 5 5 4 5 23/30 Read block 45 len 36864 24/30 Read block 502 len 4096 25/30 Read block 139 len 12288 26/30 Write block 501 len 32768 data: 3 3 4 4 5 5 5 5 27/30 Write block 378 len 16384 data: 4 4 3 3 28/30 Read block 471 len 12288 29/30 Write block 485 len 40960 data: 4 3 3 3 3 3 2 2 3 3 30/30 Read block 570 len 36864 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 5 Done Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 1 26 Done 1 26 25 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:39.293533719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1334} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 29 30 30 50 53 53 F F F <--- 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 F F F 8 160-179 24 30 30 43 53 53 F F F <--- 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 28 30 30 49 53 53 F F F <--- 17 340-359 27 30 30 46 53 53 T F F <--- 18 360-379 27 27 27 46 46 46 F T T <--- 19 380-399 27 30 30 46 53 53 F T T <--- 20 400-419 27 30 30 46 53 53 F F F <--- 21 420-439 28 28 28 47 47 47 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 28 28 28 48 48 48 F T T <--- 25 500-519 27 27 27 46 46 46 F T T <--- 26 520-539 29 30 30 50 53 53 F F F <--- 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 28 28 28 47 47 47 F T T <--- Max gen: 30, Max flush: 53 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 24 24 24 43 43 43 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 18 18 18 33 33 33 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 30 30 30 53 53 53 F F F 18 360-379 31 31 31 54 54 54 F F F 19 380-399 31 31 31 54 54 54 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 31 31 31 54 54 54 F F F 26 520-539 30 30 30 53 53 53 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 31, Max flush: 54 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.618502704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630207511Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630516173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630554545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630567128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630585374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"Crucible 5bf585de-b968-4744-af38-e089b4d9f97a has session id: 88d4cd60-068b-4546-abd5-1b59ef389225","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630920274Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Upstairs opts: Upstairs UUID: 5bf585de-b968-4744-af38-e089b4d9f97a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.630955934Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Crucible stats registered with UUID: 5bf585de-b968-4744-af38-e089b4d9f97a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.6309749Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:53.630988844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631127785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631151612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631166387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631334407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631352682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"5bf585de-b968-4744-af38-e089b4d9f97a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631374149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631387233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631401407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631415301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631429295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.63144376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631465576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"ds_connection connected from Ok(127.0.0.1:48047)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631485992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"ds_connection connected from Ok(127.0.0.1:59563)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631532366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"ds_connection connected from Ok(127.0.0.1:56826)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.631659064Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.632172397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.632243188Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.632597813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.63295436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[0]R flush_numbers[0..12]: [45, 48, 53, 48, 46, 52, 42, 46, 53, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.632976846Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[0]R generation[0..12]: [26, 28, 30, 28, 27, 30, 23, 27, 30, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633003964Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633021009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[1]R flush_numbers[0..12]: [45, 48, 53, 48, 46, 52, 42, 46, 53, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633036584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[1]R generation[0..12]: [26, 28, 30, 28, 27, 30, 23, 27, 30, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633052169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633068804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[2]R flush_numbers[0..12]: [45, 48, 53, 48, 46, 52, 42, 46, 53, 43, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633084318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[2]R generation[0..12]: [26, 28, 30, 28, 27, 30, 23, 27, 30, 24, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633100853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633120519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:15:53.633134153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633153799Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633167503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633180717Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"":"downstairs","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633194711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633207225Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"0","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633221519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"1","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Transition from WaitQuorum to Active","v":0,"name"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"crucible","level":30,"time":"2026-03-17T14:15:53.633238594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"client":"2","":"downstairs","session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633252638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633265732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"5bf585de-b968-4744-af38-e089b4d9f97a is now active with session: 88d4cd60-068b-4546-abd5-1b59ef389225","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633280797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633293331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343,"session_id":"88d4cd60-068b-4546-abd5-1b59ef389225"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633306745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} {"msg":"Activated sub_volume 5bf585de-b968-4744-af38-e089b4d9f97a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:15:53.633318288Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 28672 data: 3 4 3 3 3 3 6 02/30 Write block 325 len 16384 data: 3 3 2 2 03/30 Read block 425 len 28672 04/30 Write block 187 len 24576 data: 5 5 4 4 6 5 05/30 Read block 320 len 8192 06/30 Write block 264 len 24576 data: 4 4 4 7 6 5 07/30 Read block 124 len 40960 08/30 Write block 423 len 8192 data: 4 4 09/30 Read block 99 len 28672 10/30 Read block 70 len 28672 11/30 Write block 435 len 20480 data: 5 5 4 2 2 12/30 Write block 522 len 8192 data: 3 3 13/30 Read block 286 len 24576 14/30 Write block 105 len 12288 data: 6 4 5 15/30 Write block 500 len 32768 data: 3 4 4 5 5 6 6 6 16/30 Write block 7 len 28672 data: 2 2 2 3 3 3 3 17/30 Read block 360 len 24576 18/30 Flush 19/30 Write block 353 len 16384 data: 6 6 7 6 20/30 Write block 256 len 4096 data: 3 21/30 Read block 319 len 36864 22/30 Read block 595 len 12288 23/30 Read block 507 len 12288 24/30 Read block 4 len 28672 25/30 Read block 591 len 36864 26/30 Read block 210 len 32768 27/30 Read block 119 len 24576 28/30 Write block 156 len 12288 data: 2 2 2 29/30 Read block 161 len 40960 30/30 Write block 527 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 29 4 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:15:53.645304509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1343} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 26 32 55 45 55 F F F <--- 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 32 30 32 55 52 55 F F F <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 27 27 27 46 46 46 T F T <--- 8 160-179 30 30 30 53 53 53 F F F 9 180-199 32 24 32 55 43 55 F T F <--- 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 22 22 22 40 40 40 T F T <--- 13 260-279 32 18 32 55 33 55 F T F <--- 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 32 30 32 55 53 55 F T F <--- 17 340-359 30 30 30 53 53 53 T F T <--- 18 360-379 31 31 31 54 54 54 F F F 19 380-399 31 31 31 54 54 54 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 28 32 55 47 55 F F F <--- 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 32 31 32 55 54 55 F F F <--- 26 520-539 32 30 32 55 53 55 T F T <--- 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 32, Max flush: 55 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 32 32 32 55 55 55 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 28 28 47 47 47 F F F 12 240-259 33 33 33 56 56 56 F F F 13 260-279 32 32 32 55 55 55 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 33 33 33 56 56 56 F F F 18 360-379 31 31 31 54 54 54 F F F 19 380-399 31 31 31 54 54 54 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 32 32 55 55 55 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 32 32 32 55 55 55 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 24 24 24 43 43 43 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 33, Max flush: 56 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.96539066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:07.976114455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.976171392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.98666872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.986958556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.986996558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987008641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987026396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"Crucible 696a9f9a-e329-4ad4-a627-78ca7eaca744 has session id: f887e8b9-975a-47c5-831e-9e6f997e64fd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987387754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Upstairs opts: Upstairs UUID: 696a9f9a-e329-4ad4-a627-78ca7eaca744, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987423635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Crucible stats registered with UUID: 696a9f9a-e329-4ad4-a627-78ca7eaca744","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987438779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:07.987451783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987634898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987671279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987687003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987806009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987824324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"696a9f9a-e329-4ad4-a627-78ca7eaca744 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987846111Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987859035Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987873149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987886743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987900847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987941879Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987965986Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65050)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.987981711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51043)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.9881471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41397)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.988167947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.988785421Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.988803356Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.988838536Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989200694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[0]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 55, 42, 56, 53, 55, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989222311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[0]R generation[0..12]: [32, 28, 30, 28, 27, 32, 23, 33, 30, 32, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989240676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989257651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[1]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 55, 42, 56, 53, 55, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989273106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[1]R generation[0..12]: [32, 28, 30, 28, 27, 32, 23, 33, 30, 32, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989288781Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989304995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[2]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 55, 42, 56, 53, 55, 45, 47]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.98932039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[2]R generation[0..12]: [32, 28, 30, 28, 27, 32, 23, 33, 30, 32, 26, 28]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989336005Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.98935256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989366164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:16:07.989379858Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989398533Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989413128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"":"downstairs","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989427532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989440436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"0","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.9894564Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"1","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989470505Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"client":"2","":"downstairs","session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989485069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989498593Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"696a9f9a-e329-4ad4-a627-78ca7eaca744 is now active with session: f887e8b9-975a-47c5-831e-9e6f997e64fd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989517069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989529922Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352,"session_id":"f887e8b9-975a-47c5-831e-9e6f997e64fd"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.989543276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} {"msg":"Activated sub_volume 696a9f9a-e329-4ad4-a627-78ca7eaca744","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:07.98955472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 24576 data: 6 5 6 6 5 5 02/30 Write block 220 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Write block 504 len 16384 data: 6 7 7 7 04/30 Read block 337 len 12288 05/30 Read block 576 len 32768 06/30 Read block 38 len 4096 07/30 Read block 327 len 40960 08/30 Read block 302 len 28672 09/30 Write block 102 len 40960 data: 7 6 6 7 5 6 5 5 5 3 10/30 Write block 355 len 28672 data: 8 7 6 5 3 3 4 11/30 Write block 248 len 40960 data: 3 3 4 4 3 2 2 3 4 4 12/30 Write block 550 len 32768 data: 5 5 4 3 3 3 4 4 13/30 Write block 485 len 32768 data: 5 4 4 4 4 4 3 3 14/30 Flush 15/30 Write block 100 len 4096 data: 5 16/30 Read block 411 len 32768 17/30 Read block 311 len 28672 18/30 Write block 471 len 40960 data: 4 4 4 4 5 5 4 5 5 5 19/30 Flush 20/30 Write block 254 len 32768 data: 3 4 5 5 4 4 4 4 21/30 Read block 53 len 12288 22/30 Read block 470 len 12288 23/30 Write block 511 len 28672 data: 4 4 5 5 4 3 4 24/30 Read block 367 len 4096 25/30 Read block 586 len 36864 26/30 Read block 372 len 24576 27/30 Read block 15 len 28672 28/30 Read block 179 len 40960 29/30 Read block 338 len 32768 30/30 Read block 291 len 20480 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:08.003240328Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 32 34 34 55 58 58 F F F <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 32 34 34 55 57 57 T F F <--- 10 200-219 26 26 26 45 45 45 F F F 11 220-239 28 34 34 47 57 57 T F F <--- 12 240-259 33 34 34 56 57 57 F T T <--- 13 260-279 32 32 32 55 55 55 F T T <--- 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 33 34 34 56 57 57 F F F <--- 18 360-379 31 34 34 54 57 57 F F F <--- 19 380-399 31 31 31 54 54 54 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 32 32 55 55 55 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 25 34 34 44 58 58 F F F <--- 24 480-499 31 34 34 54 58 58 F F F <--- 25 500-519 32 34 34 55 57 57 T T T <--- 26 520-539 33 33 33 56 56 56 F F F 27 540-559 24 34 34 43 57 57 F F F <--- 28 560-579 23 23 23 42 42 42 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 34, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 26 26 26 45 45 45 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 35 35 35 59 59 59 F F F 13 260-279 35 35 35 59 59 59 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 32 32 32 55 55 55 F F F 17 340-359 34 34 34 57 57 57 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 31 31 31 54 54 54 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 32 32 55 55 55 F F F 22 440-459 27 27 27 46 46 46 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 34 34 34 58 58 58 F F F 25 500-519 35 35 35 59 59 59 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 57 57 57 F F F 28 560-579 23 23 23 42 42 42 F F F 29 580-599 31 31 31 54 54 54 F F F Max gen: 35, Max flush: 59 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.327543114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.339340439Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.339701336Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.339739278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.339751971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.339769837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"Crucible 7eabe1f5-4e0e-4d65-8db8-e2677bcef551 has session id: 8007ff97-1358-4eba-a227-252350c9179e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340159793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Upstairs opts: Upstairs UUID: 7eabe1f5-4e0e-4d65-8db8-e2677bcef551, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340194103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Crucible stats registered with UUID: 7eabe1f5-4e0e-4d65-8db8-e2677bcef551","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340213199Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:22.340226753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340378868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340414489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340430344Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340559712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340575887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"7eabe1f5-4e0e-4d65-8db8-e2677bcef551 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340597994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340610887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340625062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340639146Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.34065309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340699084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340743547Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40273)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340875316Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36865)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340910187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38088)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.340926802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.341577635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.34162683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.341644325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342060989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[0]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 58, 42, 56, 53, 57, 45, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342084196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[0]R generation[0..12]: [32, 28, 30, 28, 27, 34, 23, 33, 30, 34, 26, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342099941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342116856Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[1]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 58, 42, 56, 53, 57, 45, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342132671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[1]R generation[0..12]: [32, 28, 30, 28, 27, 34, 23, 33, 30, 34, 26, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342147835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.34216416Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[2]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 58, 42, 56, 53, 57, 45, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342179565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[2]R generation[0..12]: [32, 28, 30, 28, 27, 34, 23, 33, 30, 34, 26, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342194909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342214375Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342228419Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:16:22.342242494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342261109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342274633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"":"downstairs","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342288797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342301841Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"0","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342316226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"1","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.34233024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"client":"2","":"downstairs","session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342344514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342360099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"7eabe1f5-4e0e-4d65-8db8-e2677bcef551 is now active with session: 8007ff97-1358-4eba-a227-252350c9179e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342375243Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342388437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361,"session_id":"8007ff97-1358-4eba-a227-252350c9179e"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342401451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} {"msg":"Activated sub_volume 7eabe1f5-4e0e-4d65-8db8-e2677bcef551","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:22.342412814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 4096 data: 5 02/30 Read block 252 len 40960 03/30 Read block 553 len 32768 04/30 Read block 590 len 16384 05/30 Flush 06/30 Flush 07/30 Write block 564 len 32768 data: 6 3 3 2 3 4 4 4 08/30 Write block 222 len 8192 data: 3 3 09/30 Flush 10/30 Write block 450 len 16384 data: 4 5 5 5 11/30 Flush 12/30 Write block 338 len 24576 data: 5 5 4 6 7 8 13/30 Read block 435 len 8192 14/30 Read block 562 len 24576 15/30 Read block 311 len 4096 16/30 Read block 252 len 8192 17/30 Read block 472 len 28672 18/30 Read block 118 len 36864 19/30 Read block 579 len 20480 20/30 Read block 39 len 40960 21/30 Read block 268 len 20480 22/30 Flush 23/30 Write block 188 len 28672 data: 7 6 6 7 6 5 5 24/30 Read block 491 len 40960 25/30 Write block 102 len 24576 data: 8 7 7 8 6 7 26/30 Read block 0 len 40960 27/30 Write block 202 len 28672 data: 5 5 6 4 4 4 5 28/30 Write block 592 len 8192 data: 5 6 29/30 Write block 338 len 12288 data: 6 6 5 30/30 Write block 390 len 32768 data: 3 3 2 2 3 3 3 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 4 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 21 2 25 Done 19 0 19 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1008)) Some(JobId(1021)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:22.354221743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1361} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 34 34 34 58 58 58 T F T <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 34 34 34 57 57 57 T F T <--- 10 200-219 26 26 26 45 45 45 T F T <--- 11 220-239 36 36 36 62 62 62 F F F 12 240-259 35 35 35 59 59 59 F F F 13 260-279 35 35 35 59 59 59 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 36 32 36 64 55 64 T F T <--- 17 340-359 36 34 36 64 57 64 T F T <--- 18 360-379 34 34 34 57 57 57 F F F 19 380-399 31 31 31 54 54 54 T F T <--- 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 32 32 55 55 55 F F F 22 440-459 36 27 36 63 46 63 F F F <--- 23 460-479 34 34 34 58 58 58 F F F 24 480-499 36 36 36 60 60 60 F F F 25 500-519 35 35 35 59 59 59 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 57 57 57 F F F 28 560-579 36 36 36 62 62 62 F F F 29 580-599 31 31 31 54 54 54 T F T <--- Max gen: 36, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 37 37 37 65 65 65 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 37 37 37 65 65 65 F F F 11 220-239 36 36 36 62 62 62 F F F 12 240-259 35 35 35 59 59 59 F F F 13 260-279 35 35 35 59 59 59 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 F F F 16 320-339 37 37 37 65 65 65 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 37 37 37 65 65 65 F F F 20 400-419 30 30 30 53 53 53 F F F 21 420-439 32 32 32 55 55 55 F F F 22 440-459 36 36 36 63 63 63 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 36 36 36 60 60 60 F F F 25 500-519 35 35 35 59 59 59 F F F 26 520-539 33 33 33 56 56 56 F F F 27 540-559 34 34 34 57 57 57 F F F 28 560-579 36 36 36 62 62 62 F F F 29 580-599 37 37 37 65 65 65 F F F Max gen: 37, Max flush: 65 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.673727021Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.685394457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.68570504Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.685743241Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.685758926Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.685783363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"Crucible 8864f87e-8458-4a9d-b73d-0a3a46235371 has session id: 31ee03f1-64a6-4c69-b2dd-eb684977a5ec","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686200217Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Upstairs opts: Upstairs UUID: 8864f87e-8458-4a9d-b73d-0a3a46235371, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686235218Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Crucible stats registered with UUID: 8864f87e-8458-4a9d-b73d-0a3a46235371","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686250723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:36.686263796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686421573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686456304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686471548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686597226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686614011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"8864f87e-8458-4a9d-b73d-0a3a46235371 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686630496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.68664356Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686657814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686671718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686688573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686703497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686717852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56892)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686734237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:55214)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686756463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:53998)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.686886632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.687523912Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.687596764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.687615349Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.68798644Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[0]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 65, 42, 56, 53, 65, 65, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688010867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[0]R generation[0..12]: [32, 28, 30, 28, 27, 37, 23, 33, 30, 37, 37, 36]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688027942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688044817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[1]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 65, 42, 56, 53, 65, 65, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688060472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[1]R generation[0..12]: [32, 28, 30, 28, 27, 37, 23, 33, 30, 37, 37, 36]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688078537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688095062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[2]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 65, 42, 56, 53, 65, 65, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688111067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[2]R generation[0..12]: [32, 28, 30, 28, 27, 37, 23, 33, 30, 37, 37, 36]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688126692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688143116Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.68815683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-17T14:16:36.688170965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688190921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688204995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"":"downstairs","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688219459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688232363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"0","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688246627Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"1","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688261462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"client":"2","":"downstairs","session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-17T14:16:36.688277316Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688293911Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"8864f87e-8458-4a9d-b73d-0a3a46235371 is now active with session: 31ee03f1-64a6-4c69-b2dd-eb684977a5ec","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688311577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688324721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370,"session_id":"31ee03f1-64a6-4c69-b2dd-eb684977a5ec"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688337914Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} {"msg":"Activated sub_volume 8864f87e-8458-4a9d-b73d-0a3a46235371","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:36.688349428Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 16384 data: 6 4 4 3 02/30 Read block 139 len 16384 03/30 Write block 248 len 32768 data: 4 4 5 5 4 3 4 5 04/30 Read block 483 len 40960 05/30 Write block 530 len 40960 data: 5 5 5 5 5 5 5 5 4 4 06/30 Write block 216 len 4096 data: 4 07/30 Read block 588 len 20480 08/30 Read block 302 len 16384 09/30 Flush 10/30 Write block 221 len 8192 data: 3 4 11/30 Read block 145 len 32768 12/30 Read block 416 len 24576 13/30 Read block 238 len 16384 14/30 Read block 122 len 12288 15/30 Write block 497 len 36864 data: 2 2 2 4 5 5 6 7 8 16/30 Read block 400 len 36864 17/30 Read block 75 len 8192 18/30 Read block 14 len 16384 19/30 Write block 436 len 40960 data: 6 5 3 3 2 3 3 4 4 4 20/30 Write block 271 len 20480 data: 6 6 6 4 3 21/30 Write block 323 len 36864 data: 3 3 4 4 3 3 3 4 4 22/30 Write block 175 len 24576 data: 6 6 5 4 6 6 23/30 Write block 416 len 40960 data: 3 2 2 2 2 4 4 5 5 4 24/30 Write block 254 len 20480 data: 5 6 6 6 5 25/30 Read block 549 len 32768 26/30 Write block 401 len 8192 data: 5 6 27/30 Read block 475 len 24576 28/30 Write block 303 len 36864 data: 3 4 4 3 3 4 4 3 2 29/30 Read block 452 len 12288 30/30 Write block 109 len 36864 data: 6 6 4 3 4 4 3 2 3 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 8 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:36.701123564Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1370} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 37 37 37 65 65 65 T T F <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 30 30 30 53 53 53 T T F <--- 9 180-199 37 37 37 65 65 65 T T F <--- 10 200-219 38 38 37 66 66 65 F F F <--- 11 220-239 36 36 36 62 62 62 T T F <--- 12 240-259 38 38 35 66 66 59 T T F <--- 13 260-279 35 35 35 59 59 59 T T F <--- 14 280-299 29 29 29 50 50 50 F F F 15 300-319 29 29 29 50 50 50 T T F <--- 16 320-339 37 37 37 65 65 65 T T F <--- 17 340-359 37 37 37 65 65 65 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 38 38 37 66 66 65 F F T <--- 20 400-419 30 30 30 53 53 53 T T F <--- 21 420-439 32 32 32 55 55 55 T T F <--- 22 440-459 36 36 36 63 63 63 T T F <--- 23 460-479 34 34 34 58 58 58 F F F 24 480-499 36 36 36 60 60 60 T T F <--- 25 500-519 35 35 35 59 59 59 T T F <--- 26 520-539 38 38 33 66 66 56 F F F <--- 27 540-559 34 34 34 57 57 57 F F F 28 560-579 36 36 36 62 62 62 F F F 29 580-599 37 37 37 65 65 65 F F F Max gen: 38, Max flush: 66 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 28 28 28 48 48 48 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 39 39 39 67 67 67 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 F F F 8 160-179 39 39 39 67 67 67 F F F 9 180-199 39 39 39 67 67 67 F F F 10 200-219 38 38 38 66 66 66 F F F 11 220-239 39 39 39 67 67 67 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 39 39 39 67 67 67 F F F 14 280-299 29 29 29 50 50 50 F F F 15 300-319 39 39 39 67 67 67 F F F 16 320-339 39 39 39 67 67 67 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 39 39 39 67 67 67 F F F 22 440-459 39 39 39 67 67 67 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 39 39 39 67 67 67 F F F 25 500-519 39 39 39 67 67 67 F F F 26 520-539 38 38 38 66 66 66 F F F 27 540-559 34 34 34 57 57 57 F F F 28 560-579 36 36 36 62 62 62 F F F 29 580-599 37 37 37 65 65 65 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.042432657Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.054286729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.054602073Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.054641145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.054653259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.054671264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"Crucible a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7 has session id: 4168bbcb-ef7f-4d11-97b2-52c8b75c3e43","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055075004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Upstairs opts: Upstairs UUID: a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055110865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Crucible stats registered with UUID: a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05512686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:51.055140564Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055290508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05532842Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055344294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055478735Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055493609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055508904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055521527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055535762Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055549936Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05556372Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055578084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055618856Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:65310)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055642103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:32829)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055657548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"ds_connection connected from Ok(127.0.0.1:34294)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.055754887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056442122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056463098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056482284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056868919Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[0]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 67, 42, 56, 67, 67, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056891205Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[0]R generation[0..12]: [32, 28, 30, 28, 27, 39, 23, 33, 39, 39, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05690688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056924005Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[1]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 67, 42, 56, 67, 67, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05693962Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[1]R generation[0..12]: [32, 28, 30, 28, 27, 39, 23, 33, 39, 39, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056955645Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056971899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[2]R flush_numbers[0..12]: [55, 48, 53, 48, 46, 67, 42, 56, 67, 67, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.056987024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[2]R generation[0..12]: [32, 28, 30, 28, 27, 39, 23, 33, 39, 39, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057002318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057018723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057032127Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057045881Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057064987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057080842Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"":"downstairs","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057095466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05710818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"0","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057122354Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"1","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057136188Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"client":"2","":"downstairs","session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2026-03-17T14:16:51.057150653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057167698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7 is now active with session: 4168bbcb-ef7f-4d11-97b2-52c8b75c3e43","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057182852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057195596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379,"session_id":"4168bbcb-ef7f-4d11-97b2-52c8b75c3e43"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.05720935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} {"msg":"Activated sub_volume a2a9ef9d-6c5d-410f-9d4a-b6a4762e0cc7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:16:51.057221044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 528 len 24576 data: 3 4 6 6 6 6 03/30 Write block 61 len 20480 data: 2 3 3 3 3 04/30 Read block 417 len 12288 05/30 Write block 319 len 40960 data: 3 3 3 3 4 4 5 5 4 4 06/30 Read block 353 len 32768 07/30 Write block 527 len 32768 data: 5 4 5 7 7 7 7 6 08/30 Flush 09/30 Write block 176 len 28672 data: 7 6 5 7 7 5 7 10/30 Write block 253 len 16384 data: 4 6 7 7 11/30 Read block 25 len 36864 12/30 Read block 556 len 16384 13/30 Read block 343 len 16384 14/30 Write block 287 len 28672 data: 6 6 6 6 6 6 5 15/30 Read block 107 len 16384 16/30 Write block 504 len 28672 data: 8 9 8 8 6 5 5 17/30 Write block 298 len 16384 data: 4 6 5 4 18/30 Write block 253 len 12288 data: 5 7 8 19/30 Read block 552 len 40960 20/30 Read block 422 len 8192 21/30 Flush 22/30 Write block 171 len 36864 data: 3 4 5 6 7 8 7 6 8 23/30 Write block 11 len 28672 data: 4 4 4 4 5 4 4 24/30 Write block 153 len 20480 data: 3 2 2 3 3 25/30 Read block 533 len 24576 26/30 Write block 360 len 24576 data: 4 5 4 3 2 2 27/30 Read block 467 len 36864 28/30 Write block 559 len 24576 data: 4 4 5 7 7 7 29/30 Write block 580 len 4096 data: 5 30/30 Read block 369 len 4096 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 22 22 Done 22 22 0 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:16:51.069853068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1379} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 T T F <--- 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 39 39 39 67 67 67 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 33 33 33 56 56 56 T T F <--- 8 160-179 40 40 39 70 70 67 T T F <--- 9 180-199 40 40 39 70 70 67 F F F <--- 10 200-219 38 38 38 66 66 66 F F F 11 220-239 39 39 39 67 67 67 F F F 12 240-259 40 40 39 70 70 67 F F F <--- 13 260-279 39 39 39 67 67 67 F F F 14 280-299 40 40 29 70 70 50 F F F <--- 15 300-319 40 40 40 70 70 69 F F F <--- 16 320-339 40 40 40 69 69 69 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 34 34 34 57 57 57 T T F <--- 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 39 39 39 67 67 67 F F F 22 440-459 39 39 39 67 67 67 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 39 39 39 67 67 67 F F F 25 500-519 40 40 39 70 70 67 F F F <--- 26 520-539 40 40 40 69 69 69 F F F 27 540-559 34 34 34 57 57 57 T T F <--- 28 560-579 36 36 36 62 62 62 T T F <--- 29 580-599 37 37 37 65 65 65 T T F <--- Max gen: 40, Max flush: 70 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 71 71 71 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 39 39 39 67 67 67 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 41 41 41 71 71 71 F F F 8 160-179 41 41 41 71 71 71 F F F 9 180-199 40 40 40 70 70 70 F F F 10 200-219 38 38 38 66 66 66 F F F 11 220-239 39 39 39 67 67 67 F F F 12 240-259 40 40 40 70 70 70 F F F 13 260-279 39 39 39 67 67 67 F F F 14 280-299 40 40 40 70 70 70 F F F 15 300-319 40 40 40 70 70 70 F F F 16 320-339 40 40 40 69 69 69 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 41 41 41 71 71 71 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 39 39 39 67 67 67 F F F 22 440-459 39 39 39 67 67 67 F F F 23 460-479 34 34 34 58 58 58 F F F 24 480-499 39 39 39 67 67 67 F F F 25 500-519 40 40 40 70 70 70 F F F 26 520-539 40 40 40 69 69 69 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 41 41 41 71 71 71 F F F 29 580-599 41 41 41 71 71 71 F F F Max gen: 41, Max flush: 71 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.398646983Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.410511738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.410822911Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.410862503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.410874556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.410892622Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"Crucible 62229901-71bd-427b-828b-245fb952f57b has session id: 5482c367-9d0e-4b28-b4ed-70e9f12e5a97","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411242766Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Upstairs opts: Upstairs UUID: 62229901-71bd-427b-828b-245fb952f57b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411278367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Crucible stats registered with UUID: 62229901-71bd-427b-828b-245fb952f57b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411294251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:05.411307565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411476476Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411500643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411515617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411661301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411677796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"62229901-71bd-427b-828b-245fb952f57b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411694411Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411707194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411721259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411735273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411748927Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411763071Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411777345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:49788)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411878605Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:59983)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411950137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"ds_connection connected from Ok(127.0.0.1:59213)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.411967032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.412688957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.412725298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.412742603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413110783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[0]R flush_numbers[0..12]: [71, 48, 53, 69, 46, 67, 42, 71, 71, 70, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413132379Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[0]R generation[0..12]: [41, 28, 30, 40, 27, 39, 23, 41, 41, 40, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413147794Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413164579Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[1]R flush_numbers[0..12]: [71, 48, 53, 69, 46, 67, 42, 71, 71, 70, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413180223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[1]R generation[0..12]: [41, 28, 30, 40, 27, 39, 23, 41, 41, 40, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413195978Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413212653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[2]R flush_numbers[0..12]: [71, 48, 53, 69, 46, 67, 42, 71, 71, 70, 66, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413228238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[2]R generation[0..12]: [41, 28, 30, 40, 27, 39, 23, 41, 41, 40, 38, 39]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413246463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413263058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413276872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413296788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413310532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413323906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"":"downstairs","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.41333844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413351364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"0","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413365408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"1","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413379153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"client":"2","":"downstairs","session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413393407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413409852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"62229901-71bd-427b-828b-245fb952f57b is now active with session: 5482c367-9d0e-4b28-b4ed-70e9f12e5a97","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413424896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.41343765Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388,"session_id":"5482c367-9d0e-4b28-b4ed-70e9f12e5a97"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413450904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} {"msg":"Activated sub_volume 62229901-71bd-427b-828b-245fb952f57b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:05.413464248Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 596 len 16384 data: 3 2 2 2 02/30 Read block 354 len 32768 03/30 Write block 152 len 40960 data: 3 4 3 3 4 4 3 3 2 2 04/30 Write block 195 len 24576 data: 5 7 7 7 6 5 05/30 Read block 173 len 4096 06/30 Write block 477 len 36864 data: 5 6 6 6 5 5 5 4 6 07/30 Write block 499 len 12288 data: 3 5 6 08/30 Write block 505 len 16384 data: 10 9 9 7 09/30 Read block 439 len 40960 10/30 Flush 11/30 Write block 457 len 40960 data: 3 3 3 4 4 5 5 3 5 6 12/30 Read block 570 len 20480 13/30 Write block 561 len 28672 data: 6 8 8 8 4 4 3 14/30 Read block 479 len 28672 15/30 Read block 486 len 24576 16/30 Read block 370 len 40960 17/30 Write block 6 len 40960 data: 2 3 3 3 4 5 5 5 5 6 18/30 Write block 275 len 20480 data: 4 3 3 3 2 19/30 Write block 286 len 28672 data: 7 7 7 7 7 7 7 20/30 Write block 421 len 4096 data: 5 21/30 Write block 236 len 24576 data: 3 4 5 5 5 5 22/30 Read block 178 len 4096 23/30 Flush 24/30 Write block 491 len 32768 data: 4 4 4 4 3 2 3 3 25/30 Write block 584 len 12288 data: 3 3 4 26/30 Flush 27/30 Write block 489 len 16384 data: 5 5 5 5 28/30 Write block 109 len 40960 data: 7 7 5 4 5 5 4 3 4 2 29/30 Write block 500 len 16384 data: 6 7 6 7 30/30 Read block 226 len 28672 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:05.426416187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1388} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 41 42 73 71 73 F F F <--- 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 39 39 39 67 67 67 T F T <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 42 41 42 72 71 72 F T F <--- 8 160-179 42 41 42 72 71 72 F T F <--- 9 180-199 42 40 42 72 70 72 F T F <--- 10 200-219 42 38 42 72 66 72 F T F <--- 11 220-239 42 39 42 73 67 73 F F F <--- 12 240-259 42 40 42 73 70 73 F F F <--- 13 260-279 42 39 42 73 67 73 F F F <--- 14 280-299 42 40 42 73 70 73 F F F <--- 15 300-319 40 40 40 70 70 70 F F F 16 320-339 40 40 40 69 69 69 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 41 41 41 71 71 71 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 42 39 42 73 67 73 F F F <--- 22 440-459 42 39 42 73 67 73 F F F <--- 23 460-479 42 34 42 73 58 73 F F F <--- 24 480-499 42 39 42 74 67 74 T F T <--- 25 500-519 42 40 42 72 70 72 T F T <--- 26 520-539 40 40 40 69 69 69 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 42 41 42 73 71 73 F F F <--- 29 580-599 42 41 42 74 71 74 F T F <--- Max gen: 42, Max flush: 74 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 73 73 73 F F F 1 020-039 28 28 28 48 48 48 F F F 2 040-059 30 30 30 53 53 53 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 43 43 43 75 75 75 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 42 42 42 72 72 72 F F F 8 160-179 42 42 42 72 72 72 F F F 9 180-199 42 42 42 72 72 72 F F F 10 200-219 42 42 42 72 72 72 F F F 11 220-239 42 42 42 73 73 73 F F F 12 240-259 42 42 42 73 73 73 F F F 13 260-279 42 42 42 73 73 73 F F F 14 280-299 42 42 42 73 73 73 F F F 15 300-319 40 40 40 70 70 70 F F F 16 320-339 40 40 40 69 69 69 F F F 17 340-359 37 37 37 65 65 65 F F F 18 360-379 41 41 41 71 71 71 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 42 42 42 73 73 73 F F F 24 480-499 43 43 43 75 75 75 F F F 25 500-519 43 43 43 75 75 75 F F F 26 520-539 40 40 40 69 69 69 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 42 42 42 74 74 74 F F F Max gen: 43, Max flush: 75 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.765345612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777031664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777347918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.77738619Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777398363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777416449Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"Crucible 91b736eb-78dc-4c10-b7cd-d56cacf10a9e has session id: ad2876b7-9fea-48af-bd88-1ecebf689680","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777770914Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Upstairs opts: Upstairs UUID: 91b736eb-78dc-4c10-b7cd-d56cacf10a9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777805015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Crucible stats registered with UUID: 91b736eb-78dc-4c10-b7cd-d56cacf10a9e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.777820909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:19.777834553Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.77799009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778014487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778029351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778207505Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778222699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"91b736eb-78dc-4c10-b7cd-d56cacf10a9e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778238784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778251538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778265402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778279246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.77829289Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778307584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778322219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"ds_connection connected from Ok(127.0.0.1:49211)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778368603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"ds_connection connected from Ok(127.0.0.1:33811)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778384757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"ds_connection connected from Ok(127.0.0.1:49876)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.778429991Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779128679Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779156527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779441062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779787975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[0]R flush_numbers[0..12]: [73, 48, 53, 69, 46, 75, 42, 72, 72, 72, 72, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779812643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[0]R generation[0..12]: [42, 28, 30, 40, 27, 43, 23, 42, 42, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779828648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779845082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[1]R flush_numbers[0..12]: [73, 48, 53, 69, 46, 75, 42, 72, 72, 72, 72, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779860437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[1]R generation[0..12]: [42, 28, 30, 40, 27, 43, 23, 42, 42, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779875762Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779891926Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[2]R flush_numbers[0..12]: [73, 48, 53, 69, 46, 75, 42, 72, 72, 72, 72, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779907321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[2]R generation[0..12]: [42, 28, 30, 40, 27, 43, 23, 42, 42, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779922716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779939451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779952995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779967009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779980283Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.779993707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780013363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780027687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"0","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780041881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"1","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780055685Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"client":"2","":"downstairs","session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780069919Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780083333Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"91b736eb-78dc-4c10-b7cd-d56cacf10a9e is now active with session: ad2876b7-9fea-48af-bd88-1ecebf689680","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780098068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780110692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397,"session_id":"ad2876b7-9fea-48af-bd88-1ecebf689680"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.780124226Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} {"msg":"Activated sub_volume 91b736eb-78dc-4c10-b7cd-d56cacf10a9e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:19.7801388Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 8192 data: 8 9 02/30 Read block 74 len 40960 03/30 Write block 100 len 32768 data: 6 6 9 8 8 9 7 8 04/30 Read block 411 len 16384 05/30 Write block 280 len 12288 data: 3 3 4 06/30 Write block 197 len 24576 data: 8 8 7 6 5 6 07/30 Read block 119 len 24576 08/30 Read block 464 len 32768 09/30 Read block 560 len 40960 10/30 Read block 39 len 16384 11/30 Read block 157 len 40960 12/30 Write block 23 len 8192 data: 4 4 13/30 Write block 284 len 28672 data: 6 7 8 8 8 8 8 14/30 Write block 232 len 32768 data: 2 3 3 4 4 5 6 6 15/30 Write block 354 len 32768 data: 7 9 8 7 6 4 5 6 16/30 Write block 368 len 8192 data: 2 2 17/30 Write block 34 len 24576 data: 5 5 5 5 4 4 18/30 Write block 280 len 8192 data: 4 4 19/30 Write block 42 len 32768 data: 3 4 4 4 4 5 5 5 20/30 Flush 21/30 Write block 467 len 24576 data: 6 5 6 5 5 5 22/30 Read block 267 len 16384 23/30 Read block 209 len 24576 24/30 Write block 74 len 40960 data: 2 4 5 5 7 6 5 6 6 6 25/30 Write block 588 len 8192 data: 4 3 26/30 Read block 417 len 28672 27/30 Read block 115 len 12288 28/30 Write block 232 len 40960 data: 3 4 4 5 5 6 7 7 6 6 29/30 Read block 575 len 8192 30/30 Write block 48 len 28672 data: 6 6 5 5 5 5 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:19.792560731Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1397} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 73 73 73 F F F 1 020-039 44 44 28 76 76 48 F F F <--- 2 040-059 44 44 30 76 76 53 T T F <--- 3 060-079 40 40 40 69 69 69 T T F <--- 4 080-099 27 27 27 46 46 46 T T F <--- 5 100-119 44 44 43 76 76 75 F F F <--- 6 120-139 23 23 23 42 42 42 F F F 7 140-159 42 42 42 72 72 72 F F F 8 160-179 44 44 42 76 76 72 F F T <--- 9 180-199 44 44 42 76 76 72 F F F <--- 10 200-219 44 44 42 76 76 72 F F F <--- 11 220-239 44 44 42 76 76 73 T T F <--- 12 240-259 42 42 42 73 73 73 T T F <--- 13 260-279 42 42 42 73 73 73 F F F 14 280-299 44 44 42 76 76 73 F F F <--- 15 300-319 40 40 40 70 70 70 F F F 16 320-339 40 40 40 69 69 69 F F F 17 340-359 44 44 37 76 76 65 F F F <--- 18 360-379 44 44 41 76 76 71 F F F <--- 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 42 42 42 73 73 73 T T F <--- 24 480-499 43 43 43 75 75 75 F F F 25 500-519 43 43 43 75 75 75 F F F 26 520-539 40 40 40 69 69 69 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 42 42 42 74 74 74 T T F <--- Max gen: 44, Max flush: 76 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 73 73 73 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 45 45 45 77 77 77 F F F 3 060-079 45 45 45 77 77 77 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 23 23 23 42 42 42 F F F 7 140-159 42 42 42 72 72 72 F F F 8 160-179 44 44 44 76 76 76 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 45 45 45 77 77 77 F F F 13 260-279 42 42 42 73 73 73 F F F 14 280-299 44 44 44 76 76 76 F F F 15 300-319 40 40 40 70 70 70 F F F 16 320-339 40 40 40 69 69 69 F F F 17 340-359 44 44 44 76 76 76 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 43 43 43 75 75 75 F F F 25 500-519 43 43 43 75 75 75 F F F 26 520-539 40 40 40 69 69 69 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.123433024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135190568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135509043Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135547244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135559388Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135577543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"Crucible e9d6a08c-c071-4517-9d2a-a72b01e61ca1 has session id: 3509593c-6ea4-40d4-a4ef-1f22c1f02813","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135926687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Upstairs opts: Upstairs UUID: e9d6a08c-c071-4517-9d2a-a72b01e61ca1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135965259Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Crucible stats registered with UUID: e9d6a08c-c071-4517-9d2a-a72b01e61ca1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.135981043Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:34.135994707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136188475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136224006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.13623965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136361417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136376611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"e9d6a08c-c071-4517-9d2a-a72b01e61ca1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136393576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.1364065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136420684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136434789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136448753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136471199Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136485654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"ds_connection connected from Ok(127.0.0.1:37322)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.13660745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"ds_connection connected from Ok(127.0.0.1:36916)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136654294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"ds_connection connected from Ok(127.0.0.1:49660)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.136670999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137276629Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137328535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137375679Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137740888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[0]R flush_numbers[0..12]: [73, 76, 77, 77, 77, 76, 42, 72, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137764955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[0]R generation[0..12]: [42, 44, 45, 45, 45, 44, 23, 42, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137780599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137797344Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[1]R flush_numbers[0..12]: [73, 76, 77, 77, 77, 76, 42, 72, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137813159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[1]R generation[0..12]: [42, 44, 45, 45, 45, 44, 23, 42, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137828414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137844959Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[2]R flush_numbers[0..12]: [73, 76, 77, 77, 77, 76, 42, 72, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137861063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[2]R generation[0..12]: [42, 44, 45, 45, 45, 44, 23, 42, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137879469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137896514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137910238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-17T14:17:34.137924702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137944188Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137957602Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"":"downstairs","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137972577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137985661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"0","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.137999875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"1","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138013939Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"client":"2","":"downstairs","session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138028253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138044348Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"e9d6a08c-c071-4517-9d2a-a72b01e61ca1 is now active with session: 3509593c-6ea4-40d4-a4ef-1f22c1f02813","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138059853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138072687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406,"session_id":"3509593c-6ea4-40d4-a4ef-1f22c1f02813"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138086361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} {"msg":"Activated sub_volume e9d6a08c-c071-4517-9d2a-a72b01e61ca1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:34.138097984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 8192 data: 6 6 02/30 Flush 03/30 Write block 548 len 40960 data: 6 5 6 6 5 4 4 4 5 5 04/30 Write block 272 len 40960 data: 7 7 5 5 4 4 4 3 5 5 05/30 Write block 78 len 12288 data: 8 7 6 06/30 Read block 369 len 4096 07/30 Read block 444 len 24576 08/30 Write block 273 len 28672 data: 8 6 6 5 5 5 4 09/30 Write block 5 len 28672 data: 2 3 4 4 4 5 6 10/30 Flush 11/30 Write block 150 len 24576 data: 4 4 4 5 4 4 12/30 Read block 111 len 40960 13/30 Write block 92 len 16384 data: 2 4 4 4 14/30 Write block 55 len 28672 data: 3 2 3 3 2 2 3 15/30 Write block 135 len 36864 data: 5 4 5 4 3 3 2 2 2 16/30 Write block 529 len 4096 data: 6 17/30 Write block 346 len 32768 data: 7 6 6 8 9 6 6 7 18/30 Read block 532 len 28672 19/30 Read block 467 len 8192 20/30 Read block 8 len 40960 21/30 Flush 22/30 Write block 507 len 20480 data: 10 8 6 6 5 23/30 Write block 302 len 4096 data: 4 24/30 Read block 81 len 36864 25/30 Read block 89 len 24576 26/30 Write block 318 len 12288 data: 3 4 4 27/30 Write block 344 len 12288 data: 8 6 8 28/30 Write block 430 len 4096 data: 2 29/30 Read block 68 len 36864 30/30 Write block 423 len 16384 data: 6 6 5 3 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:34.149721818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1406} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 42 46 79 73 79 F F F <--- 1 020-039 44 44 44 76 76 76 F F F 2 040-059 46 45 46 80 77 80 F F F <--- 3 060-079 46 45 46 80 77 80 F F F <--- 4 080-099 46 45 46 80 77 80 F F F <--- 5 100-119 44 44 44 76 76 76 F F F 6 120-139 46 23 46 80 42 80 F F F <--- 7 140-159 46 42 46 80 72 80 F F F <--- 8 160-179 44 44 44 76 76 76 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 45 45 45 77 77 77 F F F 13 260-279 46 42 46 79 73 79 F F F <--- 14 280-299 46 44 46 79 76 79 F F F <--- 15 300-319 40 40 40 70 70 70 T F T <--- 16 320-339 40 40 40 69 69 69 T F T <--- 17 340-359 46 44 46 80 76 80 T F T <--- 18 360-379 44 44 44 76 76 76 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 42 42 42 73 73 73 T F T <--- 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 46 43 46 78 75 78 F T F <--- 25 500-519 43 43 43 75 75 75 T F T <--- 26 520-539 46 40 46 80 69 80 F F F <--- 27 540-559 46 41 46 79 71 79 F F F <--- 28 560-579 42 42 42 73 73 73 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 46, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 46 46 46 80 80 80 F F F 3 060-079 46 46 46 80 80 80 F F F 4 080-099 46 46 46 80 80 80 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 46 46 46 80 80 80 F F F 7 140-159 46 46 46 80 80 80 F F F 8 160-179 44 44 44 76 76 76 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 45 45 45 77 77 77 F F F 13 260-279 46 46 46 79 79 79 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 47 47 47 81 81 81 F F F 16 320-339 47 47 47 81 81 81 F F F 17 340-359 47 47 47 81 81 81 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 47 47 47 81 81 81 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 46 46 46 80 80 80 F F F 27 540-559 46 46 46 79 79 79 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 47, Max flush: 81 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.496576844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.50988738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510315798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510362952Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510375895Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510395321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"Crucible e1e3fade-0bdc-46e3-bed3-3989f9c8e33f has session id: 2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510822899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Upstairs opts: Upstairs UUID: e1e3fade-0bdc-46e3-bed3-3989f9c8e33f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51086173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Crucible stats registered with UUID: e1e3fade-0bdc-46e3-bed3-3989f9c8e33f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.510878635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:48.510892799Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511066101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511277514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511304932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511522487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511539472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"e1e3fade-0bdc-46e3-bed3-3989f9c8e33f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.5115649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511580684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511595589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511610133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511641933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511658448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511673682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60758)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511692228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43176)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511707792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33189)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.511723577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.512462687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.512491016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.512508371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51297606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[0]R flush_numbers[0..12]: [79, 76, 80, 80, 80, 76, 80, 80, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513021524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[0]R generation[0..12]: [46, 44, 46, 46, 46, 44, 46, 46, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51304266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513059475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[1]R flush_numbers[0..12]: [79, 76, 80, 80, 80, 76, 80, 80, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51307536Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[1]R generation[0..12]: [46, 44, 46, 46, 46, 44, 46, 46, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513091225Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51310801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[2]R flush_numbers[0..12]: [79, 76, 80, 80, 80, 76, 80, 80, 76, 76, 76, 77]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513124424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[2]R generation[0..12]: [46, 44, 46, 46, 46, 44, 46, 46, 44, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513141229Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513157814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513171418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513185873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513205178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513218843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"":"downstairs","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513233517Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513246711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"0","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513261045Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"1","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.51327755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"client":"2","":"downstairs","session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:17:48.513292474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513308249Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"e1e3fade-0bdc-46e3-bed3-3989f9c8e33f is now active with session: 2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513323784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513336878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415,"session_id":"2b8cf9e3-fad7-4469-9c0b-5c7c1753c6a7"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513350502Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} {"msg":"Activated sub_volume e1e3fade-0bdc-46e3-bed3-3989f9c8e33f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:17:48.513362015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 237 len 20480 data: 7 8 8 7 7 02/30 Write block 372 len 16384 data: 4 5 5 4 03/30 Write block 140 len 40960 data: 4 3 3 3 2 2 2 3 3 4 04/30 Write block 38 len 8192 data: 5 5 05/30 Flush 06/30 Write block 530 len 16384 data: 8 8 8 8 07/30 Write block 141 len 16384 data: 4 4 4 3 08/30 Write block 560 len 4096 data: 5 09/30 Read block 243 len 32768 10/30 Read block 243 len 16384 11/30 Write block 147 len 16384 data: 4 4 5 5 12/30 Write block 522 len 32768 data: 4 4 3 3 3 6 5 7 13/30 Read block 78 len 12288 14/30 Write block 533 len 40960 data: 9 7 6 6 6 5 5 3 4 3 15/30 Write block 243 len 32768 data: 5 5 3 3 3 5 5 6 16/30 Read block 278 len 4096 17/30 Write block 165 len 20480 data: 2 2 2 2 2 18/30 Read block 587 len 32768 19/30 Read block 299 len 28672 20/30 Read block 49 len 40960 21/30 Write block 82 len 32768 data: 7 7 5 4 4 3 2 2 22/30 Write block 433 len 4096 data: 4 23/30 Read block 444 len 16384 24/30 Write block 48 len 36864 data: 7 7 6 6 6 6 5 4 3 25/30 Write block 310 len 8192 data: 4 3 26/30 Write block 288 len 36864 data: 9 9 9 8 8 6 6 6 6 27/30 Read block 510 len 40960 28/30 Read block 236 len 32768 29/30 Read block 377 len 4096 30/30 Write block 27 len 32768 data: 3 2 4 5 4 5 5 6 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:17:48.525872403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1415} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 48 48 44 82 82 76 T T F <--- 2 040-059 46 46 46 80 80 80 T T F <--- 3 060-079 46 46 46 80 80 80 F F F 4 080-099 46 46 46 80 80 80 T T F <--- 5 100-119 44 44 44 76 76 76 F F F 6 120-139 46 46 46 80 80 80 F F F 7 140-159 48 48 46 82 82 80 T T F <--- 8 160-179 44 44 44 76 76 76 T T F <--- 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 45 82 82 77 F F T <--- 12 240-259 48 48 45 82 82 77 T T T <--- 13 260-279 46 46 46 79 79 79 F F F 14 280-299 46 46 46 79 79 79 T T F <--- 15 300-319 47 47 47 81 81 81 T T F <--- 16 320-339 47 47 47 81 81 81 F F F 17 340-359 47 47 47 81 81 81 F F F 18 360-379 48 48 44 82 82 76 F F F <--- 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 47 47 47 81 81 81 T T F <--- 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 46 46 46 80 80 80 T T F <--- 27 540-559 46 46 46 79 79 79 T T F <--- 28 560-579 42 42 42 73 73 73 T T F <--- 29 580-599 45 45 45 77 77 77 F F F Max gen: 48, Max flush: 82 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 49 49 49 83 83 83 F F F 3 060-079 46 46 46 80 80 80 F F F 4 080-099 49 49 49 83 83 83 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 46 46 46 80 80 80 F F F 7 140-159 49 49 49 83 83 83 F F F 8 160-179 49 49 49 83 83 83 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 49 49 49 83 83 83 F F F 13 260-279 46 46 46 79 79 79 F F F 14 280-299 49 49 49 83 83 83 F F F 15 300-319 49 49 49 83 83 83 F F F 16 320-339 47 47 47 81 81 81 F F F 17 340-359 47 47 47 81 81 81 F F F 18 360-379 48 48 48 82 82 82 F F F 19 380-399 38 38 38 66 66 66 F F F 20 400-419 39 39 39 67 67 67 F F F 21 420-439 49 49 49 83 83 83 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 46 46 46 78 78 78 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 49 49 49 83 83 83 F F F 27 540-559 49 49 49 83 83 83 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 45 45 45 77 77 77 F F F Max gen: 49, Max flush: 83 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.903832276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91552585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.915837223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.915875304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.915887298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.915905603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"Crucible 252037dd-d7a8-4922-a76c-0b91ea422e9f has session id: 7bae4c65-ed06-43ba-a81c-bc807751615e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916239763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Upstairs opts: Upstairs UUID: 252037dd-d7a8-4922-a76c-0b91ea422e9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916312595Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Crucible stats registered with UUID: 252037dd-d7a8-4922-a76c-0b91ea422e9f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91633112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:02.916346255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916535301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916570732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916592928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916720076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916737411Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"252037dd-d7a8-4922-a76c-0b91ea422e9f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916752586Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91676558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916779734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916793678Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916807462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91686623Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91690103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51446)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.916932809Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59924)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.917051355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50140)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.917097739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.917653144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.917682653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.917704629Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91814256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[0]R flush_numbers[0..12]: [79, 83, 83, 80, 83, 76, 80, 83, 83, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918164556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[0]R generation[0..12]: [46, 49, 49, 46, 49, 44, 46, 49, 49, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918180631Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918196976Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[1]R flush_numbers[0..12]: [79, 83, 83, 80, 83, 76, 80, 83, 83, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918212341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[1]R generation[0..12]: [46, 49, 49, 46, 49, 44, 46, 49, 49, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918227495Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.91824337Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[2]R flush_numbers[0..12]: [79, 83, 83, 80, 83, 76, 80, 83, 83, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918258414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[2]R generation[0..12]: [46, 49, 49, 46, 49, 44, 46, 49, 49, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918273839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918293425Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918307359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:18:02.918321393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918339639Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918352823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"":"downstairs","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918366937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918379461Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"0","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918393225Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"1","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918406819Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"client":"2","":"downstairs","session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918421053Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918437818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"252037dd-d7a8-4922-a76c-0b91ea422e9f is now active with session: 7bae4c65-ed06-43ba-a81c-bc807751615e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918453032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918465886Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424,"session_id":"7bae4c65-ed06-43ba-a81c-bc807751615e"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.9184793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} {"msg":"Activated sub_volume 252037dd-d7a8-4922-a76c-0b91ea422e9f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:02.918490944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 257 len 24576 data: 7 6 5 5 5 5 03/30 Write block 582 len 8192 data: 4 5 04/30 Read block 24 len 32768 05/30 Write block 380 len 20480 data: 4 4 2 2 2 06/30 Flush 07/30 Write block 153 len 28672 data: 6 5 5 5 5 4 4 08/30 Read block 352 len 36864 09/30 Read block 310 len 16384 10/30 Read block 115 len 28672 11/30 Write block 72 len 24576 data: 2 2 3 5 6 6 12/30 Write block 122 len 32768 data: 3 3 3 3 4 4 5 4 13/30 Read block 123 len 36864 14/30 Write block 487 len 40960 data: 6 5 6 6 6 6 5 5 4 3 15/30 Read block 383 len 8192 16/30 Read block 586 len 24576 17/30 Write block 148 len 12288 data: 5 6 6 18/30 Write block 163 len 24576 data: 2 2 3 3 3 3 19/30 Read block 122 len 20480 20/30 Read block 539 len 32768 21/30 Read block 295 len 20480 22/30 Read block 359 len 16384 23/30 Read block 481 len 16384 24/30 Write block 416 len 20480 data: 4 3 3 3 3 25/30 Read block 220 len 36864 26/30 Read block 539 len 32768 27/30 Read block 463 len 20480 28/30 Read block 360 len 4096 29/30 Read block 179 len 40960 30/30 Read block 193 len 8192 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) None Some(JobId(1005)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:02.931245864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 49 49 49 83 83 83 F F F 3 060-079 46 46 46 80 80 80 T F T <--- 4 080-099 49 49 49 83 83 83 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 46 46 46 80 80 80 T F T <--- 7 140-159 49 49 49 83 83 83 T F T <--- 8 160-179 49 49 49 83 83 83 T F T <--- 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 49 50 85 83 85 F F F <--- 13 260-279 50 46 50 85 79 85 F F F <--- 14 280-299 49 49 49 83 83 83 F F F 15 300-319 49 49 49 83 83 83 F F F 16 320-339 47 47 47 81 81 81 F F F 17 340-359 47 47 47 81 81 81 F F F 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 38 50 85 66 85 F F F <--- 20 400-419 39 39 39 67 67 67 T F T <--- 21 420-439 49 49 49 83 83 83 T F T <--- 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 46 46 46 78 78 78 T F T <--- 25 500-519 47 47 47 81 81 81 F F F 26 520-539 49 49 49 83 83 83 F F F 27 540-559 49 49 49 83 83 83 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 50 45 50 85 77 85 F F F <--- Max gen: 50, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 49 49 49 83 83 83 F F F 3 060-079 51 51 51 86 86 86 F F F 4 080-099 49 49 49 83 83 83 F F F 5 100-119 44 44 44 76 76 76 F F F 6 120-139 51 51 51 86 86 86 F F F 7 140-159 51 51 51 86 86 86 F F F 8 160-179 51 51 51 86 86 86 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 48 48 82 82 82 F F F 12 240-259 50 50 50 85 85 85 F F F 13 260-279 50 50 50 85 85 85 F F F 14 280-299 49 49 49 83 83 83 F F F 15 300-319 49 49 49 83 83 83 F F F 16 320-339 47 47 47 81 81 81 F F F 17 340-359 47 47 47 81 81 81 F F F 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 50 50 85 85 85 F F F 20 400-419 51 51 51 86 86 86 F F F 21 420-439 51 51 51 86 86 86 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 51 51 51 86 86 86 F F F 25 500-519 47 47 47 81 81 81 F F F 26 520-539 49 49 49 83 83 83 F F F 27 540-559 49 49 49 83 83 83 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 50 50 50 85 85 85 F F F Max gen: 51, Max flush: 86 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.253252805Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:17.263989394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.264045401Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.274464055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.274752731Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.274793964Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.274805677Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.274823893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"Crucible 76842f1c-2a97-401d-a73e-04185de6c818 has session id: bc312e08-fff6-46a5-9f38-655c69095555","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275179859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Upstairs opts: Upstairs UUID: 76842f1c-2a97-401d-a73e-04185de6c818, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275214939Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Crucible stats registered with UUID: 76842f1c-2a97-401d-a73e-04185de6c818","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275229994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:17.275243328Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275399444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275424332Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275439066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275590011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275605956Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"76842f1c-2a97-401d-a73e-04185de6c818 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275629663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275642887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275657181Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275671315Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275684979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275699614Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275713868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"ds_connection connected from Ok(127.0.0.1:64915)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275738955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"ds_connection connected from Ok(127.0.0.1:52797)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.27575425Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"ds_connection connected from Ok(127.0.0.1:40975)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.275769114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.276499342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.27652784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.276566842Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.276944745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[0]R flush_numbers[0..12]: [79, 83, 83, 86, 83, 76, 86, 86, 86, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.276967101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[0]R generation[0..12]: [46, 49, 49, 51, 49, 44, 51, 51, 51, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.276985717Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277002392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[1]R flush_numbers[0..12]: [79, 83, 83, 86, 83, 76, 86, 86, 86, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277017686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[1]R generation[0..12]: [46, 49, 49, 51, 49, 44, 51, 51, 51, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277033171Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277049236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[2]R flush_numbers[0..12]: [79, 83, 83, 86, 83, 76, 86, 86, 86, 76, 76, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.27706447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[2]R generation[0..12]: [46, 49, 49, 51, 49, 44, 51, 51, 51, 44, 44, 48]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277079505Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277095619Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277109214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277123268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:18:17.277136462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277155117Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"":"downstairs","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277169271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277182085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"0","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.27719813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"1","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277212224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"client":"2","":"downstairs","session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277226158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:18:17.277239532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"76842f1c-2a97-401d-a73e-04185de6c818 is now active with session: bc312e08-fff6-46a5-9f38-655c69095555","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277256037Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277268941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433,"session_id":"bc312e08-fff6-46a5-9f38-655c69095555"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277282075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} {"msg":"Activated sub_volume 76842f1c-2a97-401d-a73e-04185de6c818","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:17.277293729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 4096 data: 7 02/30 Read block 32 len 36864 03/30 Write block 110 len 16384 data: 8 6 5 6 04/30 Read block 160 len 28672 05/30 Write block 510 len 32768 data: 7 6 5 6 6 5 4 5 06/30 Read block 215 len 8192 07/30 Write block 233 len 28672 data: 5 5 6 6 8 9 9 08/30 Write block 152 len 20480 data: 5 7 6 6 6 09/30 Read block 432 len 8192 10/30 Read block 393 len 8192 11/30 Read block 386 len 12288 12/30 Read block 31 len 16384 13/30 Write block 52 len 24576 data: 7 7 6 5 4 4 14/30 Read block 160 len 20480 15/30 Read block 251 len 28672 16/30 Flush 17/30 Write block 405 len 20480 data: 4 3 3 3 2 18/30 Write block 327 len 12288 data: 5 5 4 19/30 Read block 125 len 32768 20/30 Read block 62 len 4096 21/30 Flush 22/30 Write block 266 len 36864 data: 5 8 7 6 5 7 8 9 7 23/30 Write block 523 len 24576 data: 5 4 4 4 7 6 24/30 Read block 512 len 32768 25/30 Read block 68 len 32768 26/30 Read block 64 len 28672 27/30 Read block 298 len 36864 28/30 Read block 77 len 4096 29/30 Write block 308 len 40960 data: 5 5 5 4 2 2 2 3 3 3 30/30 Read block 573 len 16384 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:17.289907457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1433} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 49 52 52 83 87 87 F F F <--- 3 060-079 51 51 51 86 86 86 F F F 4 080-099 49 49 49 83 83 83 F F F 5 100-119 44 52 52 76 87 87 T F F <--- 6 120-139 51 51 51 86 86 86 F F F 7 140-159 51 52 52 86 87 87 F F F <--- 8 160-179 51 51 51 86 86 86 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 48 52 52 82 87 87 F F F <--- 12 240-259 50 50 50 85 85 85 F F F 13 260-279 50 50 50 85 85 85 F T T <--- 14 280-299 49 49 49 83 83 83 F F F 15 300-319 49 49 49 83 83 83 F T T <--- 16 320-339 47 52 52 81 88 88 F F F <--- 17 340-359 47 52 52 81 87 87 T F F <--- 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 50 50 85 85 85 F F F 20 400-419 51 52 52 86 88 88 F F F <--- 21 420-439 51 51 51 86 86 86 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 51 51 51 86 86 86 F F F 25 500-519 47 52 52 81 87 87 F F F <--- 26 520-539 49 49 49 83 83 83 F T T <--- 27 540-559 49 49 49 83 83 83 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 50 50 50 85 85 85 F F F Max gen: 52, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 51 51 51 86 86 86 F F F 4 080-099 49 49 49 83 83 83 F F F 5 100-119 52 52 52 87 87 87 F F F 6 120-139 51 51 51 86 86 86 F F F 7 140-159 52 52 52 87 87 87 F F F 8 160-179 51 51 51 86 86 86 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 52 52 52 87 87 87 F F F 12 240-259 50 50 50 85 85 85 F F F 13 260-279 53 53 53 89 89 89 F F F 14 280-299 49 49 49 83 83 83 F F F 15 300-319 53 53 53 89 89 89 F F F 16 320-339 52 52 52 88 88 88 F F F 17 340-359 52 52 52 87 87 87 F F F 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 50 50 85 85 85 F F F 20 400-419 52 52 52 88 88 88 F F F 21 420-439 51 51 51 86 86 86 F F F 22 440-459 42 42 42 73 73 73 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 51 51 51 86 86 86 F F F 25 500-519 52 52 52 87 87 87 F F F 26 520-539 53 53 53 89 89 89 F F F 27 540-559 49 49 49 83 83 83 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 50 50 50 85 85 85 F F F Max gen: 53, Max flush: 89 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.614023137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.625749321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626070907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.62611312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626126014Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626144179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"Crucible cde0fc72-7b45-4c77-ac4d-57cd159f7334 has session id: b58522d2-47fa-4a36-ad9c-09931bf7504d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626489122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Upstairs opts: Upstairs UUID: cde0fc72-7b45-4c77-ac4d-57cd159f7334, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626522892Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Crucible stats registered with UUID: cde0fc72-7b45-4c77-ac4d-57cd159f7334","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626541667Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:31.626555311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626725782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626760573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626776037Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626930223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.62695473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"cde0fc72-7b45-4c77-ac4d-57cd159f7334 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626967224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626980218Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.626994912Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627009087Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627022661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627037015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627051239Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33575)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627066144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46108)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627192711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40881)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627215428Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627787979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627807184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.627823049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628242224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[0]R flush_numbers[0..12]: [79, 83, 87, 86, 83, 87, 86, 87, 86, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628266461Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[0]R generation[0..12]: [46, 49, 52, 51, 49, 52, 51, 52, 51, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628282216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628298721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[1]R flush_numbers[0..12]: [79, 83, 87, 86, 83, 87, 86, 87, 86, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628314196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[1]R generation[0..12]: [46, 49, 52, 51, 49, 52, 51, 52, 51, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.62832945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628345585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[2]R flush_numbers[0..12]: [79, 83, 87, 86, 83, 87, 86, 87, 86, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628360969Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[2]R generation[0..12]: [46, 49, 52, 51, 49, 52, 51, 52, 51, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628376064Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628392599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628408544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628422648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":Wait for a query_work_queue command to finish before sending IO 1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628442324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628456048Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"":"downstairs","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628470282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628483056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"0","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.62849719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"1","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628511364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"client":"2","":"downstairs","session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2026-03-17T14:18:31.628525929Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628542584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"cde0fc72-7b45-4c77-ac4d-57cd159f7334 is now active with session: b58522d2-47fa-4a36-ad9c-09931bf7504d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628557808Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628570592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442,"session_id":"b58522d2-47fa-4a36-ad9c-09931bf7504d"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628583916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} {"msg":"Activated sub_volume cde0fc72-7b45-4c77-ac4d-57cd159f7334","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:31.628595499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 36864 data: 7 8 9 9 7 9 7 8 9 02/30 Write block 97 len 36864 data: 5 5 5 7 7 10 9 9 10 03/30 Write block 86 len 32768 data: 5 4 3 3 2 2 3 5 04/30 Write block 491 len 20480 data: 7 7 6 6 5 05/30 Write block 318 len 32768 data: 4 5 5 4 4 5 5 6 06/30 Write block 313 len 12288 data: 3 3 4 07/30 Read block 435 len 4096 08/30 Write block 580 len 32768 data: 6 5 5 6 4 4 5 4 09/30 Write block 416 len 28672 data: 5 4 4 4 4 6 5 10/30 Read block 310 len 8192 11/30 Write block 356 len 8192 data: 9 8 12/30 Write block 1 len 12288 data: 2 2 2 13/30 Write block 486 len 28672 data: 5 7 6 7 7 8 8 14/30 Write block 97 len 16384 data: 6 6 6 8 15/30 Read block 7 len 24576 16/30 Write block 135 len 40960 data: 6 5 6 5 4 5 5 5 5 4 17/30 Write block 294 len 4096 data: 7 18/30 Write block 173 len 16384 data: 6 7 9 10 19/30 Read block 409 len 28672 20/30 Write block 293 len 24576 data: 7 8 7 7 5 5 21/30 Read block 381 len 12288 22/30 Read block 252 len 8192 23/30 Write block 444 len 12288 data: 5 5 3 24/30 Read block 16 len 4096 25/30 Read block 482 len 28672 26/30 Read block 165 len 28672 27/30 Write block 63 len 36864 data: 4 4 4 4 4 5 4 4 2 28/30 Read block 306 len 4096 29/30 Read block 452 len 16384 30/30 Write block 545 len 20480 data: 4 5 6 7 6 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:31.639516863Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1442} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 T T F <--- 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 51 51 51 86 86 86 T T F <--- 4 080-099 49 49 49 83 83 83 T T F <--- 5 100-119 52 52 52 87 87 87 T T F <--- 6 120-139 51 51 51 86 86 86 T T F <--- 7 140-159 52 52 52 87 87 87 T T F <--- 8 160-179 51 51 51 86 86 86 T T F <--- 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 52 52 52 87 87 87 F F F 12 240-259 50 50 50 85 85 85 F F F 13 260-279 53 53 53 89 89 89 F F F 14 280-299 49 49 49 83 83 83 T T F <--- 15 300-319 53 53 53 89 89 89 T T F <--- 16 320-339 52 52 52 88 88 88 T T F <--- 17 340-359 52 52 52 87 87 87 T T F <--- 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 50 50 85 85 85 F F F 20 400-419 52 52 52 88 88 88 T T F <--- 21 420-439 51 51 51 86 86 86 T T F <--- 22 440-459 42 42 42 73 73 73 T T F <--- 23 460-479 45 45 45 77 77 77 F F F 24 480-499 51 51 51 86 86 86 T T F <--- 25 500-519 52 52 52 87 87 87 F F F 26 520-539 53 53 53 89 89 89 F F F 27 540-559 49 49 49 83 83 83 T T F <--- 28 560-579 49 49 49 83 83 83 F F F 29 580-599 50 50 50 85 85 85 T T F <--- Max gen: 53, Max flush: 89 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 90 90 90 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 54 54 54 90 90 90 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 54 54 90 90 90 F F F 7 140-159 54 54 54 90 90 90 F F F 8 160-179 54 54 54 90 90 90 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F F F 11 220-239 52 52 52 87 87 87 F F F 12 240-259 50 50 50 85 85 85 F F F 13 260-279 53 53 53 89 89 89 F F F 14 280-299 54 54 54 90 90 90 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 48 48 48 82 82 82 F F F 19 380-399 50 50 50 85 85 85 F F F 20 400-419 54 54 54 90 90 90 F F F 21 420-439 54 54 54 90 90 90 F F F 22 440-459 54 54 54 90 90 90 F F F 23 460-479 45 45 45 77 77 77 F F F 24 480-499 54 54 54 90 90 90 F F F 25 500-519 52 52 52 87 87 87 F F F 26 520-539 53 53 53 89 89 89 F F F 27 540-559 54 54 54 90 90 90 F F F 28 560-579 49 49 49 83 83 83 F F F 29 580-599 54 54 54 90 90 90 F F F Max gen: 54, Max flush: 90 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:45.981673019Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:45.99238285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:45.992426383Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.002904275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003246207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003285999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003298243Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003316448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"Crucible 4ce87a79-4de0-4eb9-b33c-1996cea9c405 has session id: 8a33385e-2f5b-4a64-9652-6a2efb3c6b9a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003680507Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Upstairs opts: Upstairs UUID: 4ce87a79-4de0-4eb9-b33c-1996cea9c405, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003711146Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Crucible stats registered with UUID: 4ce87a79-4de0-4eb9-b33c-1996cea9c405","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00372643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:46.003739774Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003907984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003932802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.003947836Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004107694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004124329Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"4ce87a79-4de0-4eb9-b33c-1996cea9c405 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004141764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004154778Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004169112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004183116Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00419682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.0042274Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004273273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36506)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00429746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54672)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.004313175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59313)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00432846Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005047824Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005076963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005094318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005574761Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[0]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 90, 90, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005598228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[0]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 54, 54, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005615883Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005632838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[1]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 90, 90, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005648763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[1]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 54, 54, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005664278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005680653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[2]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 90, 90, 76, 76, 87]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005696027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[2]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 54, 54, 44, 44, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005711692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005728047Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Max found gen is 55","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005741601Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Generation requested: 56 >= found:55","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005755775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 91","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005775211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005788795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"":"downstairs","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005802979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005815523Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"0","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005831558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"1","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005846062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"client":"2","":"downstairs","session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005860606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00587429Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"4ce87a79-4de0-4eb9-b33c-1996cea9c405 is now active with session: 8a33385e-2f5b-4a64-9652-6a2efb3c6b9a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005892866Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.00590584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451,"session_id":"8a33385e-2f5b-4a64-9652-6a2efb3c6b9a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005920004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} {"msg":"Activated sub_volume 4ce87a79-4de0-4eb9-b33c-1996cea9c405","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:18:46.005931737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 32768 data: 6 7 7 9 10 10 8 8 02/30 Write block 145 len 36864 data: 3 3 5 6 7 7 5 6 8 03/30 Read block 431 len 32768 04/30 Read block 459 len 16384 05/30 Write block 461 len 36864 data: 5 6 6 4 6 7 7 6 7 06/30 Write block 420 len 28672 data: 5 7 6 7 7 6 4 07/30 Read block 61 len 36864 08/30 Write block 378 len 20480 data: 5 5 5 5 3 09/30 Read block 550 len 4096 10/30 Read block 207 len 4096 11/30 Flush 12/30 Write block 404 len 36864 data: 4 5 4 4 4 3 2 3 3 13/30 Write block 263 len 8192 data: 4 5 14/30 Write block 208 len 36864 data: 6 6 4 4 3 3 3 3 5 15/30 Read block 316 len 36864 16/30 Read block 369 len 40960 17/30 Read block 53 len 12288 18/30 Write block 524 len 4096 data: 5 19/30 Read block 327 len 28672 20/30 Read block 27 len 36864 21/30 Read block 343 len 40960 22/30 Read block 520 len 36864 23/30 Read block 417 len 24576 24/30 Write block 567 len 16384 data: 4 4 5 5 25/30 Read block 38 len 4096 26/30 Read block 22 len 20480 27/30 Read block 562 len 4096 28/30 Write block 444 len 12288 data: 6 6 4 29/30 Read block 399 len 20480 30/30 Read block 417 len 24576 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 3 30 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:18:46.018618027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1451} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 90 90 90 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 54 54 54 90 90 90 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 54 54 90 90 90 F F F 7 140-159 54 56 56 90 91 91 T F F <--- 8 160-179 54 54 54 90 90 90 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 44 44 44 76 76 76 F T T <--- 11 220-239 52 56 56 87 91 91 T F F <--- 12 240-259 50 56 56 85 91 91 T F F <--- 13 260-279 53 53 53 89 89 89 F T T <--- 14 280-299 54 54 54 90 90 90 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 48 56 56 82 91 91 F F F <--- 19 380-399 50 56 56 85 91 91 F F F <--- 20 400-419 54 54 54 90 90 90 F T T <--- 21 420-439 54 56 56 90 91 91 F F F <--- 22 440-459 54 54 54 90 90 90 F T T <--- 23 460-479 45 56 56 77 91 91 F F F <--- 24 480-499 54 54 54 90 90 90 F F F 25 500-519 52 52 52 87 87 87 F F F 26 520-539 53 53 53 89 89 89 F T T <--- 27 540-559 54 54 54 90 90 90 F F F 28 560-579 49 49 49 83 83 83 F T T <--- 29 580-599 54 54 54 90 90 90 F F F Max gen: 56, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 90 90 90 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 54 54 54 90 90 90 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 54 54 90 90 90 F F F 7 140-159 56 56 56 91 91 91 F F F 8 160-179 54 54 54 90 90 90 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 57 57 57 92 92 92 F F F 11 220-239 56 56 56 91 91 91 F F F 12 240-259 56 56 56 91 91 91 F F F 13 260-279 57 57 57 92 92 92 F F F 14 280-299 54 54 54 90 90 90 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 56 56 56 91 91 91 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 57 57 57 92 92 92 F F F 21 420-439 56 56 56 91 91 91 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 56 56 56 91 91 91 F F F 24 480-499 54 54 54 90 90 90 F F F 25 500-519 52 52 52 87 87 87 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 54 54 54 90 90 90 F F F 28 560-579 57 57 57 92 92 92 F F F 29 580-599 54 54 54 90 90 90 F F F Max gen: 57, Max flush: 92 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.349288203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:00.360024062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.360069525Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.370359671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.370641975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.370680697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.370693131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.370711226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"Crucible 034b7646-30d9-426d-acd3-d835a71df940 has session id: ebbc4043-1427-4ff8-bf50-61dd883828bf","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371085237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Upstairs opts: Upstairs UUID: 034b7646-30d9-426d-acd3-d835a71df940, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371121758Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Crucible stats registered with UUID: 034b7646-30d9-426d-acd3-d835a71df940","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371137073Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:00.371150277Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371310545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371333801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371348676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371499541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371516246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"034b7646-30d9-426d-acd3-d835a71df940 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371532741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371545685Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371559959Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371573753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371587647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371601901Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371616226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:59374)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371669652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:45220)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371836201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:48179)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.371858138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372388186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372497478Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372563208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372892016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[0]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 91, 90, 76, 92, 91]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372914172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[0]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 56, 54, 44, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372929927Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372946312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[1]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 91, 90, 76, 92, 91]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372961757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[1]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 56, 54, 44, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.372977031Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373010531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[2]R flush_numbers[0..12]: [90, 83, 87, 90, 90, 90, 90, 91, 90, 76, 92, 91]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373026426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[2]R generation[0..12]: [54, 49, 52, 54, 54, 54, 54, 56, 54, 44, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373044811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373061426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:19:00.37307499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373094936Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.37310857Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373122104Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"":"downstairs","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373136208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373148752Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"0","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373162946Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"1","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373179371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"client":"2","":"downstairs","session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373194076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.37320772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"034b7646-30d9-426d-acd3-d835a71df940 is now active with session: ebbc4043-1427-4ff8-bf50-61dd883828bf","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373222554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373235218Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460,"session_id":"ebbc4043-1427-4ff8-bf50-61dd883828bf"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373248392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} {"msg":"Activated sub_volume 034b7646-30d9-426d-acd3-d835a71df940","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:00.373261906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 8192 data: 7 7 02/30 Read block 462 len 16384 03/30 Write block 269 len 16384 data: 7 6 8 9 04/30 Read block 420 len 4096 05/30 Write block 408 len 32768 data: 5 4 3 4 4 3 4 3 06/30 Flush 07/30 Write block 273 len 4096 data: 10 08/30 Write block 554 len 24576 data: 5 5 6 6 5 5 09/30 Read block 438 len 36864 10/30 Read block 427 len 12288 11/30 Read block 546 len 40960 12/30 Write block 580 len 16384 data: 7 6 6 7 13/30 Read block 515 len 32768 14/30 Flush 15/30 Write block 560 len 4096 data: 6 16/30 Write block 506 len 32768 data: 10 11 9 7 8 7 6 7 17/30 Write block 472 len 8192 data: 6 5 18/30 Write block 224 len 40960 data: 3 3 3 3 3 2 2 2 4 6 19/30 Read block 164 len 20480 20/30 Flush 21/30 Write block 246 len 16384 data: 4 4 6 6 22/30 Read block 462 len 4096 23/30 Write block 256 len 4096 data: 8 24/30 Write block 489 len 24576 data: 8 8 9 9 7 7 25/30 Read block 457 len 4096 26/30 Read block 478 len 12288 27/30 Read block 250 len 32768 28/30 Write block 266 len 28672 data: 6 9 8 8 7 9 10 29/30 Read block 112 len 12288 30/30 Write block 286 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:00.384672366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1460} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 90 90 90 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 54 58 58 90 93 93 T F F <--- 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 54 54 90 90 90 F F F 7 140-159 56 56 56 91 91 91 F F F 8 160-179 54 54 54 90 90 90 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 57 57 57 92 92 92 F F F 11 220-239 56 58 58 91 95 95 F F F <--- 12 240-259 56 56 56 91 91 91 F T T <--- 13 260-279 57 58 58 92 94 94 T T T <--- 14 280-299 54 54 54 90 90 90 F T T <--- 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 56 56 56 91 91 91 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 57 58 58 92 93 93 F F F <--- 21 420-439 56 56 56 91 91 91 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 56 58 58 91 95 95 F F F <--- 24 480-499 54 54 54 90 90 90 F T T <--- 25 500-519 52 58 58 87 95 95 F F F <--- 26 520-539 57 57 57 92 92 92 F F F 27 540-559 54 58 58 90 94 94 F F F <--- 28 560-579 57 58 58 92 95 95 F F F <--- 29 580-599 54 58 58 90 94 94 F F F <--- Max gen: 58, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 90 90 90 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 58 58 58 93 93 93 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 54 54 90 90 90 F F F 7 140-159 56 56 56 91 91 91 F F F 8 160-179 54 54 54 90 90 90 F F F 9 180-199 44 44 44 76 76 76 F F F 10 200-219 57 57 57 92 92 92 F F F 11 220-239 58 58 58 95 95 95 F F F 12 240-259 59 59 59 96 96 96 F F F 13 260-279 59 59 59 96 96 96 F F F 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 54 54 90 90 90 F F F 18 360-379 56 56 56 91 91 91 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 58 58 58 93 93 93 F F F 21 420-439 56 56 56 91 91 91 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 F F F 29 580-599 58 58 58 94 94 94 F F F Max gen: 59, Max flush: 96 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.714932538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:14.725666126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.725718302Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73598111Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736260433Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736298514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736310598Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736328974Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"Crucible 25ec893b-ad18-4f48-a4e0-869ff0159b23 has session id: a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736676867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Upstairs opts: Upstairs UUID: 25ec893b-ad18-4f48-a4e0-869ff0159b23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736707056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Crucible stats registered with UUID: 25ec893b-ad18-4f48-a4e0-869ff0159b23","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736722061Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:14.736735145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736905815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736929472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.736944097Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737097753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737112917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"25ec893b-ad18-4f48-a4e0-869ff0159b23 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737130652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737143376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73715735Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737171074Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737184859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737214687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737230042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44544)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737281547Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56505)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737435273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51537)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.737463492Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738034582Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73806296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738079865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738529709Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[0]R flush_numbers[0..12]: [90, 83, 87, 90, 93, 90, 90, 91, 90, 76, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738551626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[0]R generation[0..12]: [54, 49, 52, 54, 58, 54, 54, 56, 54, 44, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73856726Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738583845Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[1]R flush_numbers[0..12]: [90, 83, 87, 90, 93, 90, 90, 91, 90, 76, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73859926Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[1]R generation[0..12]: [54, 49, 52, 54, 58, 54, 54, 56, 54, 44, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738614785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73863135Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[2]R flush_numbers[0..12]: [90, 83, 87, 90, 93, 90, 90, 91, 90, 76, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738646424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[2]R generation[0..12]: [54, 49, 52, 54, 58, 54, 54, 56, 54, 44, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738661689Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738678123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738691447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738705452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:19:14.738718876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738736801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"":"downstairs","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738753066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.73876611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"0","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738780294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"1","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738794068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"client":"2","":"downstairs","session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738808442Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-17T14:19:14.738822186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"25ec893b-ad18-4f48-a4e0-869ff0159b23 is now active with session: a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738839682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738852405Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469,"session_id":"a372eb62-fdec-440a-ab9f-b6bd8cb2d6dd"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738865569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} {"msg":"Activated sub_volume 25ec893b-ad18-4f48-a4e0-869ff0159b23","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:14.738876843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 8192 data: 8 10 02/30 Read block 295 len 20480 03/30 Write block 144 len 4096 data: 5 04/30 Read block 162 len 4096 05/30 Write block 197 len 16384 data: 9 9 8 7 06/30 Write block 346 len 4096 data: 11 07/30 Write block 360 len 32768 data: 6 7 5 4 3 3 2 2 08/30 Write block 0 len 28672 data: 2 3 3 3 2 3 4 09/30 Write block 426 len 12288 data: 5 4 3 10/30 Read block 146 len 20480 11/30 Write block 129 len 40960 data: 5 4 4 5 5 5 7 6 7 6 12/30 Read block 439 len 24576 13/30 Read block 262 len 24576 14/30 Read block 383 len 32768 15/30 Read block 389 len 40960 16/30 Read block 304 len 8192 17/30 Flush 18/30 Write block 174 len 16384 data: 8 10 11 8 19/30 Read block 44 len 24576 20/30 Read block 39 len 8192 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 352 len 16384 data: 7 8 8 10 25/30 Read block 187 len 24576 26/30 Read block 278 len 16384 27/30 Read block 287 len 32768 28/30 Read block 246 len 20480 29/30 Read block 198 len 24576 30/30 Write block 244 len 36864 data: 6 4 5 5 7 7 7 6 5 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 1 24 Done 8 29 29 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:14.750908588Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1469} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 60 60 90 97 97 T F F <--- 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 58 58 58 93 93 93 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 54 60 60 90 97 97 F F F <--- 7 140-159 56 60 60 91 97 97 T F F <--- 8 160-179 54 60 60 90 98 98 F F F <--- 9 180-199 44 60 60 76 97 97 T F F <--- 10 200-219 57 60 60 92 97 97 T F F <--- 11 220-239 58 58 58 95 95 95 F F F 12 240-259 59 59 59 96 96 96 F T T <--- 13 260-279 59 59 59 96 96 96 F F F 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 54 60 60 90 97 97 T T T <--- 18 360-379 56 60 60 91 97 97 T F F <--- 19 380-399 56 56 56 91 91 91 F F F 20 400-419 58 58 58 93 93 93 F F F 21 420-439 56 60 60 91 97 97 F F F <--- 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 F F F 29 580-599 58 58 58 94 94 94 F F F Max gen: 60, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 F F F 1 020-039 49 49 49 83 83 83 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F F F 4 080-099 58 58 58 93 93 93 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 60 60 60 97 97 97 F F F 8 160-179 60 60 60 98 98 98 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 60 60 60 97 97 97 F F F 11 220-239 58 58 58 95 95 95 F F F 12 240-259 61 61 61 99 99 99 F F F 13 260-279 59 59 59 96 96 96 F F F 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 58 58 58 93 93 93 F F F 21 420-439 60 60 60 97 97 97 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 F F F 29 580-599 58 58 58 94 94 94 F F F Max gen: 61, Max flush: 99 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.06992888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:29.08063434Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.080673222Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.090959427Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091247473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091285264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091297717Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091315793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"Crucible 374cc393-ad6f-4e35-84b4-663f34d7cd9c has session id: 80afd275-3fa7-4fe1-b2ce-a53d36b12d1d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091622914Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Upstairs opts: Upstairs UUID: 374cc393-ad6f-4e35-84b4-663f34d7cd9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091670269Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Crucible stats registered with UUID: 374cc393-ad6f-4e35-84b4-663f34d7cd9c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091686543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:29.091700177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091869968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091904768Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.091920233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092054743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092069427Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"374cc393-ad6f-4e35-84b4-663f34d7cd9c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092086603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092099256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092113481Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092127755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092141439Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092156083Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.09217884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45024)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092196575Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33156)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.09221207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43535)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092227184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092958872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.092988521Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093012759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093425271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[0]R flush_numbers[0..12]: [97, 83, 87, 90, 93, 90, 97, 97, 98, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093450589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[0]R generation[0..12]: [60, 49, 52, 54, 58, 54, 60, 60, 60, 60, 60, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093466724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093483589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[1]R flush_numbers[0..12]: [97, 83, 87, 90, 93, 90, 97, 97, 98, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093499253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[1]R generation[0..12]: [60, 49, 52, 54, 58, 54, 60, 60, 60, 60, 60, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093514658Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093531533Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[2]R flush_numbers[0..12]: [97, 83, 87, 90, 93, 90, 97, 97, 98, 97, 97, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093550319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[2]R generation[0..12]: [60, 49, 52, 54, 58, 54, 60, 60, 60, 60, 60, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093565933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093582578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093596202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093616628Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093630182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093643737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"":"downstairs","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093658271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093671075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"0","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093685339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"1","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093699423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"client":"2","":"downstairs","session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:19:29.093714168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093729672Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"374cc393-ad6f-4e35-84b4-663f34d7cd9c is now active with session: 80afd275-3fa7-4fe1-b2ce-a53d36b12d1d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093744617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.09375734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478,"session_id":"80afd275-3fa7-4fe1-b2ce-a53d36b12d1d"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093772855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} {"msg":"Activated sub_volume 374cc393-ad6f-4e35-84b4-663f34d7cd9c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:29.093784839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 217 len 40960 data: 3 2 2 3 4 5 4 4 4 4 02/30 Read block 186 len 4096 03/30 Read block 389 len 32768 04/30 Write block 89 len 36864 data: 4 3 3 4 6 5 5 5 7 05/30 Write block 431 len 20480 data: 3 4 5 4 6 06/30 Write block 250 len 4096 data: 8 07/30 Flush 08/30 Write block 414 len 32768 data: 5 4 6 5 5 5 6 8 09/30 Flush 10/30 Write block 435 len 4096 data: 7 11/30 Write block 83 len 28672 data: 8 6 5 6 5 4 5 12/30 Flush 13/30 Write block 270 len 4096 data: 8 14/30 Read block 337 len 28672 15/30 Read block 172 len 20480 16/30 Flush 17/30 Flush 18/30 Write block 140 len 4096 data: 6 19/30 Read block 125 len 40960 20/30 Flush 21/30 Write block 589 len 12288 data: 4 6 6 22/30 Read block 493 len 8192 23/30 Read block 51 len 16384 24/30 Write block 400 len 8192 data: 5 6 25/30 Read block 13 len 24576 26/30 Read block 116 len 32768 27/30 Read block 464 len 28672 28/30 Read block 226 len 12288 29/30 Write block 15 len 36864 data: 7 5 5 4 4 2 3 3 5 30/30 Write block 62 len 32768 data: 4 5 5 5 5 5 6 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 19 2 2 23 Done 2 19 19 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:29.105746773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1478} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 97 97 97 F T T <--- 1 020-039 49 49 49 83 83 83 F T T <--- 2 040-059 52 52 52 87 87 87 F F F 3 060-079 54 54 54 90 90 90 F T T <--- 4 080-099 62 62 62 100 102 102 T F F <--- 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 60 62 62 97 105 105 F F F <--- 8 160-179 60 60 60 98 98 98 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 62 62 62 100 100 100 F F F 11 220-239 62 62 62 100 100 100 F F F 12 240-259 62 62 62 100 100 100 F F F 13 260-279 59 62 62 96 103 103 F F F <--- 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 62 62 62 101 101 101 F T T <--- 21 420-439 62 62 62 101 102 102 T F F <--- 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 F F F 29 580-599 58 58 58 94 94 94 F T T <--- Max gen: 62, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 106 106 106 F F F 1 020-039 63 63 63 106 106 106 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 62 62 62 102 102 102 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 60 60 60 98 98 98 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 62 62 62 100 100 100 F F F 11 220-239 62 62 62 100 100 100 F F F 12 240-259 62 62 62 100 100 100 F F F 13 260-279 62 62 62 103 103 103 F F F 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 56 56 56 91 91 91 F F F 20 400-419 63 63 63 106 106 106 F F F 21 420-439 62 62 62 102 102 102 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 F F F 26 520-539 57 57 57 92 92 92 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 63, Max flush: 106 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.422833721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.434688564Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435029575Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435069687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435082261Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435100506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"Crucible 68e62f18-2f1d-4216-ae50-26be9a2b5042 has session id: 7f6dd296-c668-492d-9632-003a85c44f34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435433396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Upstairs opts: Upstairs UUID: 68e62f18-2f1d-4216-ae50-26be9a2b5042, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435468906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Crucible stats registered with UUID: 68e62f18-2f1d-4216-ae50-26be9a2b5042","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435485001Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:43.435498265Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435678339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435708908Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435723992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435867835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435882409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"68e62f18-2f1d-4216-ae50-26be9a2b5042 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435904716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435920251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435935045Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435949489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435963674Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.435978238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436015669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"ds_connection connected from Ok(127.0.0.1:63551)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436068435Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"ds_connection connected from Ok(127.0.0.1:64787)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436163883Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"ds_connection connected from Ok(127.0.0.1:46909)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436184379Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436812237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436854509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.436871794Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437306494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[0]R flush_numbers[0..12]: [106, 106, 87, 106, 102, 90, 97, 105, 98, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437330351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[0]R generation[0..12]: [63, 63, 52, 63, 62, 54, 60, 62, 60, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437346336Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437366412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[1]R flush_numbers[0..12]: [106, 106, 87, 106, 102, 90, 97, 105, 98, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437382737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[1]R generation[0..12]: [63, 63, 52, 63, 62, 54, 60, 62, 60, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437398011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437414606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[2]R flush_numbers[0..12]: [106, 106, 87, 106, 102, 90, 97, 105, 98, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437429991Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[2]R generation[0..12]: [63, 63, 52, 63, 62, 54, 60, 62, 60, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437445465Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.43746156Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437475554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437489888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:19:43.437503152Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437522558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"":"downstairs","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437536862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437549696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"0","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437564021Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"1","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437580105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"client":"2","":"downstairs","session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.43759467Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437608704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"68e62f18-2f1d-4216-ae50-26be9a2b5042 is now active with session: 7f6dd296-c668-492d-9632-003a85c44f34","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437627419Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437640473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487,"session_id":"7f6dd296-c668-492d-9632-003a85c44f34"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437654327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} {"msg":"Activated sub_volume 68e62f18-2f1d-4216-ae50-26be9a2b5042","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:43.437666081Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 259 len 8192 data: 6 6 02/30 Write block 170 len 12288 data: 2 4 5 03/30 Read block 418 len 8192 04/30 Write block 9 len 4096 data: 5 05/30 Read block 533 len 24576 06/30 Read block 466 len 28672 07/30 Write block 92 len 4096 data: 5 08/30 Write block 528 len 4096 data: 7 09/30 Write block 385 len 40960 data: 7 5 5 4 3 4 4 3 3 4 10/30 Flush 11/30 Write block 560 len 8192 data: 7 7 12/30 Write block 272 len 12288 data: 11 11 8 13/30 Read block 459 len 32768 14/30 Read block 590 len 24576 15/30 Read block 62 len 12288 16/30 Read block 90 len 20480 17/30 Write block 507 len 36864 data: 12 10 8 9 8 7 8 7 6 18/30 Read block 433 len 24576 19/30 Read block 428 len 40960 20/30 Read block 176 len 20480 21/30 Read block 521 len 16384 22/30 Read block 152 len 8192 23/30 Read block 153 len 16384 24/30 Read block 434 len 40960 25/30 Read block 516 len 12288 26/30 Write block 394 len 36864 data: 5 4 4 4 4 5 6 7 7 27/30 Read block 354 len 16384 28/30 Write block 300 len 36864 data: 6 5 5 4 5 5 4 4 6 29/30 Read block 450 len 40960 30/30 Read block 459 len 16384 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:43.45014871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1487} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 63 64 107 106 107 F F F <--- 1 020-039 63 63 63 106 106 106 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 62 64 107 102 107 F F F <--- 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 64 60 64 107 98 107 F F F <--- 9 180-199 60 60 60 97 97 97 F F F 10 200-219 62 62 62 100 100 100 F F F 11 220-239 62 62 62 100 100 100 F F F 12 240-259 64 62 64 107 100 107 F T F <--- 13 260-279 64 62 64 107 103 107 T T T <--- 14 280-299 59 59 59 96 96 96 F F F 15 300-319 54 54 54 90 90 90 T F T <--- 16 320-339 54 54 54 90 90 90 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 64 56 64 107 91 107 T F T <--- 20 400-419 63 63 63 106 106 106 T F T <--- 21 420-439 62 62 62 102 102 102 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 58 58 58 95 95 95 T F T <--- 26 520-539 64 57 64 107 92 107 F F F <--- 27 540-559 58 58 58 94 94 94 F F F 28 560-579 58 58 58 95 95 95 T F T <--- 29 580-599 63 63 63 106 106 106 F F F Max gen: 64, Max flush: 107 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 107 107 107 F F F 1 020-039 63 63 63 106 106 106 F F F 2 040-059 52 52 52 87 87 87 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 64 64 64 107 107 107 F F F 9 180-199 60 60 60 97 97 97 F F F 10 200-219 62 62 62 100 100 100 F F F 11 220-239 62 62 62 100 100 100 F F F 12 240-259 64 64 64 107 107 107 F F F 13 260-279 65 65 65 108 108 108 F F F 14 280-299 59 59 59 96 96 96 F F F 15 300-319 65 65 65 108 108 108 F F F 16 320-339 54 54 54 90 90 90 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 65 65 65 108 108 108 F F F 20 400-419 65 65 65 108 108 108 F F F 21 420-439 62 62 62 102 102 102 F F F 22 440-459 57 57 57 92 92 92 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 65 65 65 108 108 108 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 65, Max flush: 108 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.774096732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:57.784795309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.784848155Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.795300309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.795590125Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.795628777Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79564093Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.795659006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"Crucible a3c1b293-4041-4ce5-a61d-de52e08a8f89 has session id: f99f817f-ba7e-4a40-9b01-3fdc9685e86b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796022354Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Upstairs opts: Upstairs UUID: a3c1b293-4041-4ce5-a61d-de52e08a8f89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796059755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Crucible stats registered with UUID: a3c1b293-4041-4ce5-a61d-de52e08a8f89","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79607544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:57.796089244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796229706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796253793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796268607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796408539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796423734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"a3c1b293-4041-4ce5-a61d-de52e08a8f89 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796440949Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796453422Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796467307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796481301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796495055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796509709Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796524494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48148)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796573558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64892)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796592314Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51208)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.796607298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797364034Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797391982Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797408887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797771695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[0]R flush_numbers[0..12]: [107, 106, 87, 106, 107, 90, 97, 105, 107, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797793092Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[0]R generation[0..12]: [64, 63, 52, 63, 64, 54, 60, 62, 64, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797808776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797825351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[1]R flush_numbers[0..12]: [107, 106, 87, 106, 107, 90, 97, 105, 107, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797841016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[1]R generation[0..12]: [64, 63, 52, 63, 64, 54, 60, 62, 64, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79785658Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797873205Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[2]R flush_numbers[0..12]: [107, 106, 87, 106, 107, 90, 97, 105, 107, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79788916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[2]R generation[0..12]: [64, 63, 52, 63, 64, 54, 60, 62, 64, 60, 62, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797906815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79792313Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797936724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79795669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797970324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797983808Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"":"downstairs","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.797998123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798010626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"0","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798024951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"1","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798039085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"client":"2","":"downstairs","session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-17T14:19:57.798053559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798069264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"a3c1b293-4041-4ce5-a61d-de52e08a8f89 is now active with session: f99f817f-ba7e-4a40-9b01-3fdc9685e86b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798084568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798097162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496,"session_id":"f99f817f-ba7e-4a40-9b01-3fdc9685e86b"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.798110676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} {"msg":"Activated sub_volume a3c1b293-4041-4ce5-a61d-de52e08a8f89","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:19:57.79812212Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 8192 data: 6 6 02/30 Read block 199 len 12288 03/30 Write block 429 len 12288 data: 3 3 4 04/30 Read block 44 len 36864 05/30 Read block 45 len 40960 06/30 Write block 265 len 12288 data: 5 7 10 07/30 Write block 9 len 20480 data: 6 6 7 6 6 08/30 Flush 09/30 Write block 301 len 40960 data: 6 6 5 6 6 5 5 7 6 6 10/30 Flush 11/30 Write block 37 len 24576 data: 6 6 6 3 3 4 12/30 Write block 403 len 8192 data: 6 5 13/30 Flush 14/30 Write block 326 len 24576 data: 6 6 6 5 5 5 15/30 Read block 519 len 28672 16/30 Read block 549 len 16384 17/30 Read block 203 len 36864 18/30 Read block 198 len 28672 19/30 Write block 443 len 32768 data: 5 7 7 5 2 3 4 5 20/30 Write block 299 len 24576 data: 7 7 7 7 6 7 21/30 Write block 175 len 24576 data: 11 12 9 7 9 8 22/30 Read block 133 len 4096 23/30 Write block 12 len 8192 data: 7 7 24/30 Write block 235 len 20480 data: 8 8 10 11 11 25/30 Write block 229 len 28672 data: 3 3 3 5 7 7 9 26/30 Write block 42 len 36864 data: 5 5 5 5 5 6 8 8 7 27/30 Read block 393 len 36864 28/30 Read block 191 len 32768 29/30 Write block 231 len 16384 data: 4 6 8 8 30/30 Read block 570 len 32768 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:19:57.810429467Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 66 66 107 109 109 F T T <--- 1 020-039 63 66 66 106 111 111 F F F <--- 2 040-059 52 66 66 87 111 111 F T T <--- 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 64 64 64 107 107 107 F T T <--- 9 180-199 60 60 60 97 97 97 F T T <--- 10 200-219 62 62 62 100 100 100 F F F 11 220-239 62 62 62 100 100 100 F T T <--- 12 240-259 64 64 64 107 107 107 F F F 13 260-279 65 66 66 108 109 109 F F F <--- 14 280-299 59 59 59 96 96 96 F T T <--- 15 300-319 65 66 66 108 110 110 F T T <--- 16 320-339 54 54 54 90 90 90 F T T <--- 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 65 66 66 108 109 109 T F F <--- 20 400-419 65 66 66 108 111 111 F F F <--- 21 420-439 62 66 66 102 109 109 F F F <--- 22 440-459 57 57 57 92 92 92 F T T <--- 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 65 65 65 108 108 108 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 66, Max flush: 111 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 112 112 112 F F F 1 020-039 66 66 66 111 111 111 F F F 2 040-059 67 67 67 112 112 112 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 54 54 54 90 90 90 F F F 6 120-139 60 60 60 97 97 97 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 67 67 67 112 112 112 F F F 9 180-199 67 67 67 112 112 112 F F F 10 200-219 62 62 62 100 100 100 F F F 11 220-239 67 67 67 112 112 112 F F F 12 240-259 64 64 64 107 107 107 F F F 13 260-279 66 66 66 109 109 109 F F F 14 280-299 67 67 67 112 112 112 F F F 15 300-319 67 67 67 112 112 112 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 F F F 19 380-399 66 66 66 109 109 109 F F F 20 400-419 66 66 66 111 111 111 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 67 67 67 112 112 112 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 F F F 25 500-519 65 65 65 108 108 108 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 58 58 58 94 94 94 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 67, Max flush: 112 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.144020051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.155883026Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.15619945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156237802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156249705Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156267611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"Crucible 69bb5bc6-4e91-4c1a-874d-c0d200985536 has session id: 05260ad0-89b6-4449-b400-3cc2cbdb5494","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.15663395Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Upstairs opts: Upstairs UUID: 69bb5bc6-4e91-4c1a-874d-c0d200985536, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156697639Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Crucible stats registered with UUID: 69bb5bc6-4e91-4c1a-874d-c0d200985536","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156724777Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:12.156738991Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156900099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156925207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.156940081Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.15707103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157087885Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"69bb5bc6-4e91-4c1a-874d-c0d200985536 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157106161Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157118944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157132859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157146873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157160497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157175021Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157189825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"ds_connection connected from Ok(127.0.0.1:44644)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157257436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"ds_connection connected from Ok(127.0.0.1:58891)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157286374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"ds_connection connected from Ok(127.0.0.1:36051)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157302319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157972119Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.157999917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158019083Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158368227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[0]R flush_numbers[0..12]: [112, 111, 112, 106, 107, 90, 97, 105, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158389773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[0]R generation[0..12]: [67, 66, 67, 63, 64, 54, 60, 62, 67, 67, 62, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158408329Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158425194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[1]R flush_numbers[0..12]: [112, 111, 112, 106, 107, 90, 97, 105, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158441108Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[1]R generation[0..12]: [67, 66, 67, 63, 64, 54, 60, 62, 67, 67, 62, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158456333Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158472608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[2]R flush_numbers[0..12]: [112, 111, 112, 106, 107, 90, 97, 105, 112, 112, 100, 112]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158488182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[2]R generation[0..12]: [67, 66, 67, 63, 64, 54, 60, 62, 67, 67, 62, 67]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158503517Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158519772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158533296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.15854715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158566936Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.1585806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"":"downstairs","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158594924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158607548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"0","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158623473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"1","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158637297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"client":"2","":"downstairs","session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158651391Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158664795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 69bb5bc6-4e91-4c1a-874d-c0d200985536 is now active with session: 05260ad0-89b6-4449-b400-3cc2cbdb5494","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158679749Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158692883Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505,"session_id":"05260ad0-89b6-4449-b400-3cc2cbdb5494"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158706327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} {"msg":"Activated sub_volume 69bb5bc6-4e91-4c1a-874d-c0d200985536","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:12.158717961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 12288 data: 6 6 6 02/30 Read block 585 len 32768 03/30 Write block 396 len 32768 data: 5 5 5 6 7 8 8 7 04/30 Write block 219 len 16384 data: 3 4 5 6 05/30 Read block 215 len 20480 06/30 Write block 554 len 20480 data: 6 6 7 7 6 07/30 Flush 08/30 Write block 237 len 24576 data: 11 12 12 9 9 5 09/30 Read block 470 len 24576 10/30 Write block 11 len 36864 data: 8 8 8 6 8 6 6 5 5 11/30 Read block 572 len 12288 12/30 Read block 338 len 8192 13/30 Read block 571 len 36864 14/30 Write block 113 len 40960 data: 7 6 5 4 5 3 2 2 3 4 15/30 Read block 33 len 8192 16/30 Read block 259 len 20480 17/30 Write block 197 len 36864 data: 10 10 9 8 6 7 6 7 5 18/30 Read block 79 len 40960 19/30 Read block 247 len 40960 20/30 Read block 423 len 32768 21/30 Write block 490 len 16384 data: 9 10 10 8 22/30 Write block 25 len 32768 data: 4 3 4 3 5 6 5 6 23/30 Read block 480 len 32768 24/30 Write block 49 len 8192 data: 9 8 25/30 Read block 320 len 16384 26/30 Write block 509 len 40960 data: 9 10 9 8 9 8 7 5 6 4 27/30 Read block 592 len 8192 28/30 Write block 363 len 8192 data: 5 4 29/30 Read block 209 len 4096 30/30 Read block 513 len 24576 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 27 28 Done 29 30 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:12.172428636Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1505} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 112 112 112 T T F <--- 1 020-039 66 66 66 111 111 111 T T F <--- 2 040-059 67 67 67 112 112 112 T T F <--- 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 54 54 54 90 90 90 T T F <--- 6 120-139 60 60 60 97 97 97 T T F <--- 7 140-159 62 62 62 105 105 105 F F F 8 160-179 67 67 67 112 112 112 F F F 9 180-199 67 67 67 112 112 112 T T F <--- 10 200-219 68 68 62 113 113 100 T T F <--- 11 220-239 68 68 67 113 113 112 T T F <--- 12 240-259 64 64 64 107 107 107 T T F <--- 13 260-279 66 66 66 109 109 109 F F F 14 280-299 67 67 67 112 112 112 F F F 15 300-319 67 67 67 112 112 112 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 60 60 60 97 97 97 T T F <--- 19 380-399 68 68 66 113 113 109 F F T <--- 20 400-419 68 68 66 113 113 111 F F T <--- 21 420-439 66 66 66 109 109 109 F F F 22 440-459 68 68 67 113 113 112 F F T <--- 23 460-479 58 58 58 95 95 95 F F F 24 480-499 59 59 59 96 96 96 T T F <--- 25 500-519 65 65 65 108 108 108 T T F <--- 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 58 113 113 94 F F F <--- 28 560-579 65 65 65 108 108 108 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 68, Max flush: 113 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 114 114 114 F F F 1 020-039 69 69 69 114 114 114 F F F 2 040-059 69 69 69 114 114 114 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 69 69 69 114 114 114 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 67 67 67 112 112 112 F F F 9 180-199 69 69 69 114 114 114 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 69 69 69 114 114 114 F F F 12 240-259 69 69 69 114 114 114 F F F 13 260-279 66 66 66 109 109 109 F F F 14 280-299 67 67 67 112 112 112 F F F 15 300-319 67 67 67 112 112 112 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 61 61 61 99 99 99 F F F 18 360-379 69 69 69 114 114 114 F F F 19 380-399 68 68 68 113 113 113 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 68 68 68 113 113 113 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 69 69 69 114 114 114 F F F 25 500-519 69 69 69 114 114 114 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 63 63 63 106 106 106 F F F Max gen: 69, Max flush: 114 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.511716508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.523449234Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.523757236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.523795387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.523807761Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.523827036Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"Crucible 78108a01-01c4-4013-8ad3-671f7d51af76 has session id: 31db8b29-f09a-4c06-bc93-703faf20b7d5","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524212461Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Upstairs opts: Upstairs UUID: 78108a01-01c4-4013-8ad3-671f7d51af76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524250713Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Crucible stats registered with UUID: 78108a01-01c4-4013-8ad3-671f7d51af76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524266577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:26.524280131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524428316Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524463796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524479451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524607099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524622413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"78108a01-01c4-4013-8ad3-671f7d51af76 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.52464612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524659504Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524673659Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524687723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524701437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524715701Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524730346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33805)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524766236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57611)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524785562Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41197)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.524903907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.525542337Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.525570986Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.525725192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526079728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[0]R flush_numbers[0..12]: [114, 114, 114, 106, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526105155Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[0]R generation[0..12]: [69, 69, 69, 63, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.52612122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526137945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[1]R flush_numbers[0..12]: [114, 114, 114, 106, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.52615416Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[1]R generation[0..12]: [69, 69, 69, 63, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526169614Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526186149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[2]R flush_numbers[0..12]: [114, 114, 114, 106, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526202074Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[2]R generation[0..12]: [69, 69, 69, 63, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.52622062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526237285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526250949Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526264923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:20:26.526278357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526297843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"":"downstairs","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526312327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526325151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"0","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526339175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"1","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526353109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"client":"2","":"downstairs","session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526367483Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526380827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"78108a01-01c4-4013-8ad3-671f7d51af76 is now active with session: 31db8b29-f09a-4c06-bc93-703faf20b7d5","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526398583Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526411606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514,"session_id":"31db8b29-f09a-4c06-bc93-703faf20b7d5"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526425281Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} {"msg":"Activated sub_volume 78108a01-01c4-4013-8ad3-671f7d51af76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:26.526436944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 12288 data: 8 8 6 02/30 Write block 381 len 28672 data: 7 4 3 3 8 6 6 03/30 Write block 580 len 36864 data: 8 7 7 8 5 5 6 5 5 04/30 Read block 128 len 16384 05/30 Write block 269 len 24576 data: 9 9 10 12 12 9 06/30 Write block 67 len 40960 data: 6 7 6 5 3 3 3 4 6 7 07/30 Write block 68 len 40960 data: 8 7 6 4 4 4 5 7 8 7 08/30 Read block 324 len 20480 09/30 Read block 386 len 28672 10/30 Write block 369 len 32768 data: 3 2 3 5 6 6 5 4 11/30 Read block 215 len 40960 12/30 Write block 68 len 16384 data: 9 8 7 5 13/30 Flush 14/30 Write block 370 len 16384 data: 3 4 6 7 15/30 Write block 353 len 4096 data: 9 16/30 Read block 107 len 20480 17/30 Read block 365 len 8192 18/30 Read block 433 len 28672 19/30 Write block 248 len 40960 data: 8 8 9 7 6 6 8 9 9 8 20/30 Flush 21/30 Write block 47 len 24576 data: 7 9 10 9 7 8 22/30 Write block 242 len 8192 data: 6 6 23/30 Write block 22 len 20480 data: 4 6 5 5 4 24/30 Write block 365 len 28672 data: 4 3 3 3 4 4 5 25/30 Read block 336 len 40960 26/30 Read block 326 len 16384 27/30 Write block 503 len 16384 data: 8 9 11 11 28/30 Read block 132 len 28672 29/30 Read block 394 len 40960 30/30 Read block 8 len 32768 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:26.53894351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1514} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 114 114 114 F F F 1 020-039 69 69 69 114 114 114 T F T <--- 2 040-059 69 69 69 114 114 114 T F T <--- 3 060-079 70 63 70 115 106 115 F F F <--- 4 080-099 64 64 64 107 107 107 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 69 69 69 114 114 114 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 67 67 67 112 112 112 F F F 9 180-199 69 69 69 114 114 114 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 69 69 69 114 114 114 F F F 12 240-259 70 69 70 116 114 116 T F T <--- 13 260-279 70 66 70 115 109 115 F F F <--- 14 280-299 67 67 67 112 112 112 F F F 15 300-319 67 67 67 112 112 112 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 70 61 70 116 99 116 F F F <--- 18 360-379 70 69 70 116 114 116 T F T <--- 19 380-399 70 68 70 115 113 115 F T F <--- 20 400-419 68 68 68 113 113 113 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 70 68 70 115 113 115 F T F <--- 23 460-479 58 58 58 95 95 95 F F F 24 480-499 69 69 69 114 114 114 F F F 25 500-519 69 69 69 114 114 114 T F T <--- 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 70 63 70 115 106 115 F T F <--- Max gen: 70, Max flush: 116 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 114 114 114 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 71 71 71 117 117 117 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 69 69 69 114 114 114 F F F 7 140-159 62 62 62 105 105 105 F F F 8 160-179 67 67 67 112 112 112 F F F 9 180-199 69 69 69 114 114 114 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 69 69 69 114 114 114 F F F 12 240-259 71 71 71 117 117 117 F F F 13 260-279 70 70 70 115 115 115 F F F 14 280-299 67 67 67 112 112 112 F F F 15 300-319 67 67 67 112 112 112 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 71 71 71 117 117 117 F F F 19 380-399 70 70 70 115 115 115 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 70 70 70 115 115 115 F F F 23 460-479 58 58 58 95 95 95 F F F 24 480-499 69 69 69 114 114 114 F F F 25 500-519 71 71 71 117 117 117 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 65 65 65 108 108 108 F F F 29 580-599 70 70 70 115 115 115 F F F Max gen: 71, Max flush: 117 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.863291088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87507566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875392394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875430025Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875442999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875460994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"Crucible 11d589d3-03f9-43a4-9754-ce9473f318ba has session id: 7a20b298-ab89-46db-868a-337c0f0565eb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875795274Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Upstairs opts: Upstairs UUID: 11d589d3-03f9-43a4-9754-ce9473f318ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875830344Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Crucible stats registered with UUID: 11d589d3-03f9-43a4-9754-ce9473f318ba","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875846679Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:40.875860273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.875990532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87601572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876030684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876173757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876191142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"11d589d3-03f9-43a4-9754-ce9473f318ba active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876207367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876220241Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876235035Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876249459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876263203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876278068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876292552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49858)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87641781Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:61261)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876436295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:38335)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.876500894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.877183608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.877227881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.877609535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87796359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[0]R flush_numbers[0..12]: [114, 117, 117, 115, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.877987237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[0]R generation[0..12]: [69, 71, 71, 70, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878003002Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878019677Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[1]R flush_numbers[0..12]: [114, 117, 117, 115, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878035662Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[1]R generation[0..12]: [69, 71, 71, 70, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878051066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878067501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[2]R flush_numbers[0..12]: [114, 117, 117, 115, 107, 114, 114, 105, 112, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878083196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[2]R generation[0..12]: [69, 71, 71, 70, 64, 69, 69, 62, 67, 69, 69, 69]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87809849Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878114585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878127889Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878141933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:20:40.878155297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878174963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"":"downstairs","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878189417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878202131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"0","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878218256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"1","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.87823249Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"client":"2","":"downstairs","session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878246704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878259938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"11d589d3-03f9-43a4-9754-ce9473f318ba is now active with session: 7a20b298-ab89-46db-868a-337c0f0565eb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878278154Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878291008Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523,"session_id":"7a20b298-ab89-46db-868a-337c0f0565eb"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878304432Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} {"msg":"Activated sub_volume 11d589d3-03f9-43a4-9754-ce9473f318ba","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:40.878316265Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 278 len 12288 data: 6 5 6 02/30 Write block 157 len 16384 data: 6 5 5 3 03/30 Read block 590 len 20480 04/30 Read block 133 len 12288 05/30 Write block 231 len 8192 data: 5 7 06/30 Read block 291 len 40960 07/30 Read block 33 len 20480 08/30 Read block 55 len 36864 09/30 Read block 167 len 8192 10/30 Write block 315 len 8192 data: 5 4 11/30 Read block 75 len 4096 12/30 Read block 72 len 8192 13/30 Read block 463 len 32768 14/30 Write block 250 len 4096 data: 10 15/30 Flush 16/30 Write block 121 len 16384 data: 4 5 4 4 17/30 Write block 41 len 32768 data: 4 6 6 6 6 6 8 10 18/30 Read block 426 len 32768 19/30 Flush 20/30 Write block 444 len 12288 data: 9 9 7 21/30 Read block 429 len 24576 22/30 Read block 463 len 32768 23/30 Write block 455 len 12288 data: 5 4 4 24/30 Write block 306 len 8192 data: 6 6 25/30 Write block 517 len 12288 data: 7 5 3 26/30 Write block 567 len 20480 data: 5 5 6 6 5 27/30 Write block 279 len 28672 data: 6 7 6 5 6 7 8 28/30 Read block 77 len 40960 29/30 Write block 466 len 12288 data: 8 8 7 30/30 Read block 373 len 32768 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:40.88917125Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1523} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 114 114 114 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 71 119 119 117 F F F <--- 3 060-079 70 70 70 115 115 115 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 72 72 69 119 119 114 F F F <--- 7 140-159 72 72 62 118 118 105 F F F <--- 8 160-179 72 72 67 118 118 112 F F F <--- 9 180-199 69 69 69 114 114 114 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 72 72 69 118 118 114 F F F <--- 12 240-259 72 72 71 118 118 117 F F F <--- 13 260-279 72 72 70 118 118 115 T T F <--- 14 280-299 72 72 67 118 118 112 T T F <--- 15 300-319 72 72 67 118 118 112 T T F <--- 16 320-339 67 67 67 112 112 112 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 71 71 71 117 117 117 F F F 19 380-399 70 70 70 115 115 115 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 70 70 70 115 115 115 T T F <--- 23 460-479 58 58 58 95 95 95 T T F <--- 24 480-499 69 69 69 114 114 114 F F F 25 500-519 71 71 71 117 117 117 T T F <--- 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 65 65 65 108 108 108 T T F <--- 29 580-599 70 70 70 115 115 115 F F F Max gen: 72, Max flush: 119 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 114 114 114 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 72 119 119 119 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 64 64 64 107 107 107 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 72 72 72 118 118 118 F F F 9 180-199 69 69 69 114 114 114 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 72 72 72 118 118 118 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 73 73 73 120 120 120 F F F 16 320-339 67 67 67 112 112 112 F F F 17 340-359 70 70 70 116 116 116 F F F 18 360-379 71 71 71 117 117 117 F F F 19 380-399 70 70 70 115 115 115 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 73 73 73 120 120 120 F F F 23 460-479 73 73 73 120 120 120 F F F 24 480-499 69 69 69 114 114 114 F F F 25 500-519 73 73 73 120 120 120 F F F 26 520-539 64 64 64 107 107 107 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 73 73 73 120 120 120 F F F 29 580-599 70 70 70 115 115 115 F F F Max gen: 73, Max flush: 120 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.219769654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.231498918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.231809681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.231847262Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.231859486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.231877321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"Crucible 5356f779-9399-4dc6-826e-ba8454109c3e has session id: 03ceb3c2-c1c8-4074-9f2a-7955ad0909dc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232230827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Upstairs opts: Upstairs UUID: 5356f779-9399-4dc6-826e-ba8454109c3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232268748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Crucible stats registered with UUID: 5356f779-9399-4dc6-826e-ba8454109c3e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232283552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:55.232296636Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232454593Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.2324781Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232493575Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232659844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232676409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"5356f779-9399-4dc6-826e-ba8454109c3e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232694125Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232706878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232721173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232735387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232749161Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232763745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.23277776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47050)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232795515Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34437)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232842119Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47141)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.232870997Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.233431764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.233464214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.233815779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234174086Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[0]R flush_numbers[0..12]: [114, 117, 119, 115, 107, 114, 119, 118, 118, 114, 114, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234197282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[0]R generation[0..12]: [69, 71, 72, 70, 64, 69, 72, 72, 72, 69, 69, 72]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234213917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234231323Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[1]R flush_numbers[0..12]: [114, 117, 119, 115, 107, 114, 119, 118, 118, 114, 114, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234247738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[1]R generation[0..12]: [69, 71, 72, 70, 64, 69, 72, 72, 72, 69, 69, 72]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234263082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234279367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[2]R flush_numbers[0..12]: [114, 117, 119, 115, 107, 114, 119, 118, 118, 114, 114, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234295032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[2]R generation[0..12]: [69, 71, 72, 70, 64, 69, 72, 72, 72, 69, 69, 72]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234313637Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234329962Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234343496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Generation requested: 74 >= found:74"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234364182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234377766Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.23439117Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"":"downstairs","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234405164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234417898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"0","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234432253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"1","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234446387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"client":"2","":"downstairs","session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:20:55.234461161Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234475835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"5356f779-9399-4dc6-826e-ba8454109c3e is now active with session: 03ceb3c2-c1c8-4074-9f2a-7955ad0909dc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.23449086Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234503574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532,"session_id":"03ceb3c2-c1c8-4074-9f2a-7955ad0909dc"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234516868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} {"msg":"Activated sub_volume 5356f779-9399-4dc6-826e-ba8454109c3e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:20:55.234528531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 476 len 36864 data: 6 6 7 7 7 7 7 6 5 02/30 Write block 450 len 16384 data: 7 7 7 6 03/30 Write block 414 len 8192 data: 6 5 04/30 Write block 235 len 8192 data: 10 9 05/30 Read block 311 len 40960 06/30 Read block 369 len 12288 07/30 Read block 493 len 36864 08/30 Write block 16 len 4096 data: 7 09/30 Write block 343 len 28672 data: 10 10 9 12 8 9 10 10/30 Write block 162 len 16384 data: 2 3 3 4 11/30 Write block 540 len 12288 data: 4 5 4 12/30 Read block 176 len 4096 13/30 Read block 38 len 28672 14/30 Flush 15/30 Write block 564 len 20480 data: 9 5 5 6 6 16/30 Read block 447 len 4096 17/30 Write block 584 len 20480 data: 6 6 7 6 6 18/30 Write block 316 len 24576 data: 5 4 5 6 6 5 19/30 Write block 538 len 28672 data: 6 6 5 6 5 4 5 20/30 Read block 132 len 36864 21/30 Read block 15 len 12288 22/30 Read block 350 len 32768 23/30 Read block 79 len 28672 24/30 Flush 25/30 Write block 89 len 20480 data: 6 4 4 6 7 26/30 Write block 412 len 20480 data: 5 4 7 6 7 27/30 Write block 196 len 4096 data: 8 28/30 Read block 464 len 8192 29/30 Read block 350 len 20480 30/30 Write block 472 len 24576 data: 7 6 5 6 7 7 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 29 4 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:20:55.245657237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1532} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 69 74 121 114 121 F F F <--- 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 72 119 119 119 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 64 64 64 107 107 107 T F T <--- 5 100-119 69 69 69 114 114 114 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 74 72 74 121 118 121 F F F <--- 9 180-199 69 69 69 114 114 114 T F T <--- 10 200-219 69 69 69 114 114 114 F F F 11 220-239 74 72 74 121 118 121 F T F <--- 12 240-259 72 72 72 118 118 118 F F F 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 74 73 74 122 120 122 F F F <--- 16 320-339 74 67 74 122 112 122 F F F <--- 17 340-359 74 70 74 121 116 121 F F F <--- 18 360-379 71 71 71 117 117 117 F F F 19 380-399 70 70 70 115 115 115 F F F 20 400-419 74 68 74 121 113 121 T T T <--- 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 73 74 121 120 121 F T F <--- 23 460-479 74 73 74 121 120 121 T T T <--- 24 480-499 74 69 74 121 114 121 F T F <--- 25 500-519 73 73 73 120 120 120 F F F 26 520-539 74 64 74 122 107 122 F F F <--- 27 540-559 74 68 74 122 113 122 F F F <--- 28 560-579 74 73 74 122 120 122 F F F <--- 29 580-599 74 70 74 122 115 122 F F F <--- Max gen: 74, Max flush: 122 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 121 121 121 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 72 119 119 119 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 75 75 75 123 123 123 F F F 5 100-119 69 69 69 114 114 114 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 75 75 75 123 123 123 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 74 74 74 121 121 121 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 74 74 74 122 122 122 F F F 16 320-339 74 74 74 122 122 122 F F F 17 340-359 74 74 74 121 121 121 F F F 18 360-379 71 71 71 117 117 117 F F F 19 380-399 70 70 70 115 115 115 F F F 20 400-419 75 75 75 123 123 123 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 74 74 121 121 121 F F F 23 460-479 75 75 75 123 123 123 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 73 73 73 120 120 120 F F F 26 520-539 74 74 74 122 122 122 F F F 27 540-559 74 74 74 122 122 122 F F F 28 560-579 74 74 74 122 122 122 F F F 29 580-599 74 74 74 122 122 122 F F F Max gen: 75, Max flush: 123 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.582352509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594087156Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594394817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594432519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594444612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594462998Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"Crucible 7de26309-58c8-43b8-bc08-45d0d085bd2e has session id: da1160dc-db9c-4620-ac2f-e852f528a7c0","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594850483Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Upstairs opts: Upstairs UUID: 7de26309-58c8-43b8-bc08-45d0d085bd2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594885724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Crucible stats registered with UUID: 7de26309-58c8-43b8-bc08-45d0d085bd2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.594901718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:09.594915112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595089294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595114052Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595128706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595271669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595296496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"7de26309-58c8-43b8-bc08-45d0d085bd2e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595311971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595324834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595339159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595353583Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595367247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595381711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595396096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61917)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.59541088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61158)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595549131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61548)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.595573499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596071457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596099795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596258052Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.59661992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 114, 119, 118, 121, 123, 114, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596644598Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[0]R generation[0..12]: [74, 71, 72, 70, 75, 69, 72, 72, 74, 75, 69, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596663443Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596680458Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 114, 119, 118, 121, 123, 114, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596697023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[1]R generation[0..12]: [74, 71, 72, 70, 75, 69, 72, 72, 74, 75, 69, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596712818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596729213Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 114, 119, 118, 121, 123, 114, 121]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596745088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[2]R generation[0..12]: [74, 71, 72, 70, 75, 69, 72, 72, 74, 75, 69, 74]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596760302Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596777067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596790361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596804205Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596823091Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596836345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"":"downstairs","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596850269Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596863023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"0","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596878817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"1","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596893022Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"client":"2","":"downstairs","session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:21:09.596907236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.59692228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"7de26309-58c8-43b8-bc08-45d0d085bd2e is now active with session: da1160dc-db9c-4620-ac2f-e852f528a7c0","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596937535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596950349Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541,"session_id":"da1160dc-db9c-4620-ac2f-e852f528a7c0"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596963693Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} {"msg":"Activated sub_volume 7de26309-58c8-43b8-bc08-45d0d085bd2e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:09.596975076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 32768 data: 11 10 7 8 10 9 11 10 02/30 Read block 572 len 4096 03/30 Read block 445 len 8192 04/30 Write block 328 len 8192 data: 7 6 05/30 Write block 331 len 28672 data: 6 3 4 6 5 5 4 06/30 Write block 541 len 20480 data: 7 6 5 6 5 07/30 Read block 40 len 4096 08/30 Write block 516 len 4096 data: 6 09/30 Write block 520 len 4096 data: 3 10/30 Write block 381 len 8192 data: 8 5 11/30 Write block 233 len 36864 data: 9 9 11 10 12 13 13 10 10 12/30 Read block 24 len 16384 13/30 Write block 233 len 12288 data: 10 10 12 14/30 Write block 517 len 28672 data: 8 6 4 4 4 5 6 15/30 Read block 414 len 4096 16/30 Flush 17/30 Write block 552 len 12288 data: 6 5 7 18/30 Read block 143 len 8192 19/30 Read block 95 len 24576 20/30 Write block 317 len 24576 data: 5 6 7 7 6 5 21/30 Write block 254 len 20480 data: 9 10 10 9 7 22/30 Read block 255 len 4096 23/30 Read block 269 len 36864 24/30 Read block 474 len 12288 25/30 Write block 376 len 4096 data: 5 26/30 Write block 240 len 12288 data: 11 11 7 27/30 Read block 532 len 20480 28/30 Write block 100 len 28672 data: 9 8 11 10 10 11 8 29/30 Read block 432 len 12288 30/30 Read block 18 len 28672 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:09.607990568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1541} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 121 121 121 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 72 119 119 119 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 75 75 75 123 123 123 F F F 5 100-119 69 69 69 114 114 114 T F T <--- 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 75 75 75 123 123 123 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 76 74 76 124 121 124 F F F <--- 12 240-259 76 72 76 124 118 124 T F T <--- 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 74 74 74 122 122 122 T F T <--- 16 320-339 76 74 76 124 122 124 T F T <--- 17 340-359 76 74 76 124 121 124 F T F <--- 18 360-379 71 71 71 117 117 117 T F T <--- 19 380-399 76 70 76 124 115 124 F F F <--- 20 400-419 75 75 75 123 123 123 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 74 74 121 121 121 F F F 23 460-479 75 75 75 123 123 123 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 76 73 76 124 120 124 F F F <--- 26 520-539 76 74 76 124 122 124 F F F <--- 27 540-559 76 74 76 124 122 124 T F T <--- 28 560-579 74 74 74 122 122 122 F F F 29 580-599 74 74 74 122 122 122 F F F Max gen: 76, Max flush: 124 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 121 121 121 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 72 72 72 119 119 119 F F F 3 060-079 70 70 70 115 115 115 F F F 4 080-099 75 75 75 123 123 123 F F F 5 100-119 77 77 77 125 125 125 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 72 72 72 118 118 118 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 75 75 75 123 123 123 F F F 10 200-219 69 69 69 114 114 114 F F F 11 220-239 76 76 76 124 124 124 F F F 12 240-259 77 77 77 125 125 125 F F F 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 77 77 77 125 125 125 F F F 16 320-339 77 77 77 125 125 125 F F F 17 340-359 76 76 76 124 124 124 F F F 18 360-379 77 77 77 125 125 125 F F F 19 380-399 76 76 76 124 124 124 F F F 20 400-419 75 75 75 123 123 123 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 74 74 121 121 121 F F F 23 460-479 75 75 75 123 123 123 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 76 76 76 124 124 124 F F F 27 540-559 77 77 77 125 125 125 F F F 28 560-579 74 74 74 122 122 122 F F F 29 580-599 74 74 74 122 122 122 F F F Max gen: 77, Max flush: 125 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.93150529Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943172777Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943488501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943527342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943539606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943557191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"Crucible 32b34687-d0ea-46cb-9283-bede2fb24fa6 has session id: 3d69b270-7c68-4250-ae7e-9384ceb502e1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943906145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Upstairs opts: Upstairs UUID: 32b34687-d0ea-46cb-9283-bede2fb24fa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943944567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Crucible stats registered with UUID: 32b34687-d0ea-46cb-9283-bede2fb24fa6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.943960291Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:23.943973655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944128601Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944164592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944180647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944301633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944318678Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"32b34687-d0ea-46cb-9283-bede2fb24fa6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944341365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944354699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944369533Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944383417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944397342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944411946Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.94442669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40567)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944442445Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62629)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.94446039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:55182)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.944475625Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.945237002Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.945268641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.94573402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946074741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 125, 119, 118, 121, 123, 114, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946097898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[0]R generation[0..12]: [74, 71, 72, 70, 75, 77, 72, 72, 74, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946113913Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946130618Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 125, 119, 118, 121, 123, 114, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946146603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[1]R generation[0..12]: [74, 71, 72, 70, 75, 77, 72, 72, 74, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946161957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946178402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 119, 115, 123, 125, 119, 118, 121, 123, 114, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946194187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[2]R generation[0..12]: [74, 71, 72, 70, 75, 77, 72, 72, 74, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946209261Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946228637Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946242491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:21:23.946256446Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946274411Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946287825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"":"downstairs","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946302509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946316313Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"0","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946331008Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"1","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946345242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"client":"2","":"downstairs","session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946359566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946375821Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"32b34687-d0ea-46cb-9283-bede2fb24fa6 is now active with session: 3d69b270-7c68-4250-ae7e-9384ceb502e1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946391306Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946404079Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550,"session_id":"3d69b270-7c68-4250-ae7e-9384ceb502e1"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946417443Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} {"msg":"Activated sub_volume 32b34687-d0ea-46cb-9283-bede2fb24fa6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:23.946429097Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 40960 data: 6 7 6 6 7 7 7 4 5 7 02/30 Write block 145 len 8192 data: 4 4 03/30 Read block 556 len 12288 04/30 Read block 377 len 20480 05/30 Flush 06/30 Write block 471 len 20480 data: 6 8 7 6 7 07/30 Write block 568 len 8192 data: 7 7 08/30 Write block 372 len 12288 data: 7 8 7 09/30 Read block 294 len 20480 10/30 Write block 253 len 12288 data: 7 10 11 11/30 Read block 162 len 20480 12/30 Read block 402 len 4096 13/30 Read block 218 len 24576 14/30 Read block 16 len 24576 15/30 Read block 193 len 28672 16/30 Write block 249 len 40960 data: 9 11 8 7 8 11 12 11 10 8 17/30 Write block 12 len 28672 data: 9 9 7 9 8 7 6 18/30 Write block 346 len 36864 data: 13 9 10 12 11 8 9 11 10 19/30 Write block 402 len 4096 data: 9 20/30 Flush 21/30 Write block 371 len 4096 data: 6 22/30 Write block 55 len 32768 data: 6 5 5 4 3 3 4 5 23/30 Read block 221 len 20480 24/30 Write block 196 len 32768 data: 9 11 11 10 9 7 8 7 25/30 Write block 527 len 12288 data: 8 8 8 26/30 Flush 27/30 Write block 561 len 28672 data: 8 9 9 10 6 6 7 28/30 Read block 330 len 16384 29/30 Write block 111 len 20480 data: 7 6 8 7 6 30/30 Write block 563 len 36864 data: 10 11 7 7 8 8 8 7 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 23 25 Done 24 24 2 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:23.958383909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1550} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 74 127 127 121 F F F <--- 1 020-039 78 78 78 126 126 126 F F F 2 040-059 78 78 78 128 128 126 F F F <--- 3 060-079 78 78 70 128 128 115 F F F <--- 4 080-099 75 75 75 123 123 123 F F F 5 100-119 77 77 77 125 125 125 T T F <--- 6 120-139 72 72 72 119 119 119 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 75 128 128 123 F F F <--- 10 200-219 78 78 69 128 128 114 F F F <--- 11 220-239 76 76 76 124 124 124 F F F 12 240-259 78 78 77 127 127 125 F F F <--- 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 77 77 77 125 125 125 F F F 16 320-339 77 77 77 125 125 125 F F F 17 340-359 78 78 76 127 127 124 F F F <--- 18 360-379 78 78 77 128 128 125 F F F <--- 19 380-399 76 76 76 124 124 124 F F F 20 400-419 78 78 75 127 127 123 F F F <--- 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 74 74 121 121 121 F F F 23 460-479 78 78 75 127 127 123 F F T <--- 24 480-499 74 74 74 121 121 121 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 78 78 76 128 128 124 F F F <--- 27 540-559 77 77 77 125 125 125 F F F 28 560-579 78 78 74 127 127 122 T T T <--- 29 580-599 74 74 74 122 122 122 F F F Max gen: 78, Max flush: 128 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 127 127 127 F F F 1 020-039 78 78 78 126 126 126 F F F 2 040-059 78 78 78 128 128 128 F F F 3 060-079 78 78 78 128 128 128 F F F 4 080-099 75 75 75 123 123 123 F F F 5 100-119 79 79 79 129 129 129 F F F 6 120-139 72 72 72 119 119 119 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 78 78 78 128 128 128 F F F 11 220-239 76 76 76 124 124 124 F F F 12 240-259 78 78 78 127 127 127 F F F 13 260-279 73 73 73 120 120 120 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 77 77 77 125 125 125 F F F 16 320-339 77 77 77 125 125 125 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 76 76 76 124 124 124 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 66 66 66 109 109 109 F F F 22 440-459 74 74 74 121 121 121 F F F 23 460-479 78 78 78 127 127 127 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 78 78 78 128 128 128 F F F 27 540-559 77 77 77 125 125 125 F F F 28 560-579 79 79 79 129 129 129 F F F 29 580-599 74 74 74 122 122 122 F F F Max gen: 79, Max flush: 129 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.283955733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.29559031Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.2958953Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.295933372Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.295945926Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.295963871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"Crucible b9c8e8bf-df68-4d60-96ba-51a65453ece8 has session id: d28b8be0-9a53-4f96-b270-b0201505344f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296321597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Upstairs opts: Upstairs UUID: b9c8e8bf-df68-4d60-96ba-51a65453ece8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296355828Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Crucible stats registered with UUID: b9c8e8bf-df68-4d60-96ba-51a65453ece8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296371132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:38.296385887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296529469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296553166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296567831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296715295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296730459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"b9c8e8bf-df68-4d60-96ba-51a65453ece8 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296747674Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296762799Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296776993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296791197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296805052Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296819656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.29683423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40616)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296849595Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45467)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296864899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38681)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.296879984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.297515763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.297548403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298181142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298516642Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[0]R flush_numbers[0..12]: [127, 126, 128, 128, 123, 129, 119, 126, 121, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298538898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[0]R generation[0..12]: [78, 78, 78, 78, 75, 79, 72, 78, 74, 78, 78, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298554903Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298574539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[1]R flush_numbers[0..12]: [127, 126, 128, 128, 123, 129, 119, 126, 121, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298590474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[1]R generation[0..12]: [78, 78, 78, 78, 75, 79, 72, 78, 74, 78, 78, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298606118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298622423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[2]R flush_numbers[0..12]: [127, 126, 128, 128, 123, 129, 119, 126, 121, 128, 128, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298638148Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[2]R generation[0..12]: [78, 78, 78, 78, 75, 79, 72, 78, 74, 78, 78, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298653542Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298669837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298683181Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298697235Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298717091Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298730935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"":"downstairs","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.29874532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298758424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"0","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298772878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"1","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298788923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"client":"2","":"downstairs","session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298803807Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298820182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"b9c8e8bf-df68-4d60-96ba-51a65453ece8 is now active with session: d28b8be0-9a53-4f96-b270-b0201505344f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298835166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.29884767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559,"session_id":"d28b8be0-9a53-4f96-b270-b0201505344f"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298861294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} {"msg":"Activated sub_volume b9c8e8bf-df68-4d60-96ba-51a65453ece8","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:38.298872848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 16384 data: 5 5 4 6 02/30 Write block 531 len 28672 data: 9 9 10 8 7 7 7 03/30 Read block 422 len 32768 04/30 Read block 482 len 40960 05/30 Read block 548 len 40960 06/30 Read block 66 len 28672 07/30 Write block 76 len 16384 data: 9 8 9 8 08/30 Write block 425 len 16384 data: 7 6 5 4 09/30 Write block 516 len 28672 data: 7 9 7 5 5 5 6 10/30 Write block 217 len 36864 data: 4 3 4 5 6 7 5 5 5 11/30 Read block 6 len 28672 12/30 Read block 466 len 20480 13/30 Write block 218 len 4096 data: 4 14/30 Read block 551 len 8192 15/30 Read block 161 len 28672 16/30 Write block 445 len 8192 data: 10 8 17/30 Write block 223 len 40960 data: 6 6 6 5 4 4 4 4 6 8 18/30 Read block 387 len 20480 19/30 Flush 20/30 Write block 320 len 24576 data: 8 7 6 6 6 7 21/30 Read block 376 len 4096 22/30 Write block 596 len 12288 data: 4 3 3 23/30 Read block 436 len 8192 24/30 Read block 57 len 4096 25/30 Flush 26/30 Write block 266 len 8192 data: 8 11 27/30 Write block 511 len 40960 data: 10 9 10 9 8 8 10 8 6 6 28/30 Read block 537 len 16384 29/30 Write block 461 len 36864 data: 6 7 7 5 7 9 9 8 8 30/30 Write block 132 len 16384 data: 6 6 6 8 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 20 2 24 Done 28 10 28 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:38.310649647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1559} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 127 127 127 F F F 1 020-039 80 78 80 130 126 130 F T F <--- 2 040-059 78 78 78 128 128 128 F F F 3 060-079 80 78 80 130 128 130 F T F <--- 4 080-099 75 75 75 123 123 123 F F F 5 100-119 79 79 79 129 129 129 F F F 6 120-139 72 72 72 119 119 119 T F T <--- 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 80 78 80 130 128 130 F T F <--- 11 220-239 80 76 80 130 124 130 F T F <--- 12 240-259 78 78 78 127 127 127 F F F 13 260-279 73 73 73 120 120 120 T F T <--- 14 280-299 73 73 73 120 120 120 F F F 15 300-319 77 77 77 125 125 125 F F F 16 320-339 80 77 80 131 125 131 F F F <--- 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 76 76 76 124 124 124 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 80 66 80 130 109 130 F T F <--- 22 440-459 80 74 80 130 121 130 F F F <--- 23 460-479 78 78 78 127 127 127 T F T <--- 24 480-499 74 74 74 121 121 121 F F F 25 500-519 80 76 80 130 124 130 T T T <--- 26 520-539 80 78 80 130 128 130 T T T <--- 27 540-559 77 77 77 125 125 125 F F F 28 560-579 79 79 79 129 129 129 F F F 29 580-599 80 74 80 131 122 131 F F F <--- Max gen: 80, Max flush: 131 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 127 127 127 F F F 1 020-039 80 80 80 130 130 130 F F F 2 040-059 78 78 78 128 128 128 F F F 3 060-079 80 80 80 130 130 130 F F F 4 080-099 75 75 75 123 123 123 F F F 5 100-119 79 79 79 129 129 129 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 80 80 80 130 130 130 F F F 11 220-239 80 80 80 130 130 130 F F F 12 240-259 78 78 78 127 127 127 F F F 13 260-279 81 81 81 132 132 132 F F F 14 280-299 73 73 73 120 120 120 F F F 15 300-319 77 77 77 125 125 125 F F F 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 76 76 76 124 124 124 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 80 80 80 130 130 130 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 81 81 81 132 132 132 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 81 81 81 132 132 132 F F F 27 540-559 77 77 77 125 125 125 F F F 28 560-579 79 79 79 129 129 129 F F F 29 580-599 80 80 80 131 131 131 F F F Max gen: 81, Max flush: 132 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.639291066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:52.649963576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.650018723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.660431995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.660719031Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.660756692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.660768686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.660786321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"Crucible 4b75468a-2d29-4e79-aead-5bec84bab346 has session id: f3f23724-9500-4062-9206-ef54555581cf","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.66115134Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Upstairs opts: Upstairs UUID: 4b75468a-2d29-4e79-aead-5bec84bab346, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.66118724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Crucible stats registered with UUID: 4b75468a-2d29-4e79-aead-5bec84bab346","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661202975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:52.661216619Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661365023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661401544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661417639Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661544237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661561122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"4b75468a-2d29-4e79-aead-5bec84bab346 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661576846Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.66158947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661603414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661617719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661650408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661692381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661714978Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46023)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661734653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:57455)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661750058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62073)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.661765143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662391489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662419898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662504043Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662874113Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[0]R flush_numbers[0..12]: [127, 130, 128, 130, 123, 129, 132, 126, 121, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.66289669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[0]R generation[0..12]: [78, 80, 78, 80, 75, 79, 81, 78, 74, 78, 80, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662912704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662929209Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[1]R flush_numbers[0..12]: [127, 130, 128, 130, 123, 129, 132, 126, 121, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662945204Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[1]R generation[0..12]: [78, 80, 78, 80, 75, 79, 81, 78, 74, 78, 80, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662960729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.662976984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[2]R flush_numbers[0..12]: [127, 130, 128, 130, 123, 129, 132, 126, 121, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663006292Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[2]R generation[0..12]: [78, 80, 78, 80, 75, 79, 81, 78, 74, 78, 80, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663025448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663041973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663055627Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663075893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663089207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663102751Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"":"downstairs","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663116485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663129229Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"0","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663143013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"1","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663156767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"client":"2","":"downstairs","session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663172752Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663186586Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"4b75468a-2d29-4e79-aead-5bec84bab346 is now active with session: f3f23724-9500-4062-9206-ef54555581cf","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663201681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663214394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568,"session_id":"f3f23724-9500-4062-9206-ef54555581cf"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663227698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} {"msg":"Activated sub_volume 4b75468a-2d29-4e79-aead-5bec84bab346","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:21:52.663240932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 88 len 12288 data: 5 7 5 03/30 Read block 86 len 36864 04/30 Read block 537 len 40960 05/30 Write block 259 len 20480 data: 7 7 6 6 5 06/30 Write block 467 len 28672 data: 10 9 9 6 7 9 8 07/30 Write block 209 len 40960 data: 7 5 5 4 4 4 4 6 5 5 08/30 Write block 40 len 32768 data: 5 6 8 7 7 7 7 9 09/30 Read block 222 len 12288 10/30 Write block 298 len 12288 data: 6 8 8 11/30 Read block 358 len 24576 12/30 Read block 94 len 12288 13/30 Read block 570 len 28672 14/30 Read block 149 len 40960 15/30 Write block 381 len 12288 data: 9 6 4 16/30 Read block 561 len 12288 17/30 Read block 228 len 28672 18/30 Write block 462 len 28672 data: 8 8 6 8 10 11 10 19/30 Read block 263 len 16384 20/30 Read block 474 len 20480 21/30 Write block 219 len 36864 data: 5 6 7 8 7 7 7 6 5 22/30 Read block 242 len 12288 23/30 Write block 303 len 8192 data: 7 8 24/30 Write block 47 len 8192 data: 10 11 25/30 Write block 427 len 32768 data: 6 5 4 4 5 5 6 5 26/30 Flush 27/30 Write block 553 len 8192 data: 6 8 28/30 Write block 200 len 16384 data: 10 8 9 8 29/30 Read block 247 len 20480 30/30 Write block 294 len 28672 data: 9 8 8 6 7 9 9 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:21:52.675873126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 127 127 127 F F F 1 020-039 80 80 80 130 130 130 F F F 2 040-059 78 82 82 128 134 134 F F F <--- 3 060-079 80 80 80 130 130 130 F F F 4 080-099 75 82 82 123 134 134 F F F <--- 5 100-119 79 79 79 129 129 129 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 80 82 82 130 134 134 F T T <--- 11 220-239 80 82 82 130 134 134 F F F <--- 12 240-259 78 82 82 127 134 134 F F F <--- 13 260-279 81 82 82 132 134 134 F F F <--- 14 280-299 73 82 82 120 134 134 F T T <--- 15 300-319 77 82 82 125 134 134 F T T <--- 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 76 82 82 124 134 134 F F F <--- 20 400-419 78 78 78 127 127 127 F F F 21 420-439 80 82 82 130 134 134 F F F <--- 22 440-459 80 80 80 130 130 130 F F F 23 460-479 81 82 82 132 134 134 F F F <--- 24 480-499 74 74 74 121 121 121 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 81 81 81 132 132 132 F F F 27 540-559 77 77 77 125 125 125 F T T <--- 28 560-579 79 79 79 129 129 129 F F F 29 580-599 80 80 80 131 131 131 F F F Max gen: 82, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 127 127 127 F F F 1 020-039 80 80 80 130 130 130 F F F 2 040-059 82 82 82 134 134 134 F F F 3 060-079 80 80 80 130 130 130 F F F 4 080-099 82 82 82 134 134 134 F F F 5 100-119 79 79 79 129 129 129 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 83 83 83 135 135 135 F F F 11 220-239 82 82 82 134 134 134 F F F 12 240-259 82 82 82 134 134 134 F F F 13 260-279 82 82 82 134 134 134 F F F 14 280-299 83 83 83 135 135 135 F F F 15 300-319 83 83 83 135 135 135 F F F 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 82 82 82 134 134 134 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 82 82 82 134 134 134 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 81 81 81 132 132 132 F F F 26 520-539 81 81 81 132 132 132 F F F 27 540-559 83 83 83 135 135 135 F F F 28 560-579 79 79 79 129 129 129 F F F 29 580-599 80 80 80 131 131 131 F F F Max gen: 83, Max flush: 135 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:06.998446916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.010372609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.01071102Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.010753633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.010766637Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.010784502Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"Crucible ff1d50e5-3944-4075-b07c-dc7b3890aaf9 has session id: db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011177909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Upstairs opts: Upstairs UUID: ff1d50e5-3944-4075-b07c-dc7b3890aaf9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011204587Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Crucible stats registered with UUID: ff1d50e5-3944-4075-b07c-dc7b3890aaf9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011219392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:07.011232686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011410058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011445529Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011460984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011610968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011627373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"ff1d50e5-3944-4075-b07c-dc7b3890aaf9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011659193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011673187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011687491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011701455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011715099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011729514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011744098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51147)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011809027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43236)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.011837246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:40995)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.01185335Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.01252288Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.012561652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.012969173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013352427Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[0]R flush_numbers[0..12]: [127, 130, 134, 130, 134, 129, 132, 126, 121, 128, 135, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013376754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[0]R generation[0..12]: [78, 80, 82, 80, 82, 79, 81, 78, 74, 78, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013393059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013409764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[1]R flush_numbers[0..12]: [127, 130, 134, 130, 134, 129, 132, 126, 121, 128, 135, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013425809Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[1]R generation[0..12]: [78, 80, 82, 80, 82, 79, 81, 78, 74, 78, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013441354Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013457759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[2]R flush_numbers[0..12]: [127, 130, 134, 130, 134, 129, 132, 126, 121, 128, 135, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013473513Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[2]R generation[0..12]: [78, 80, 82, 80, 82, 79, 81, 78, 74, 78, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013488588Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013504632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013518387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013532491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013552447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013565921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"":"downstairs","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013580535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013595159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"0","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013609464Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"1","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013623998Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"client":"2","":"downstairs","session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013638422Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013651856Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"ff1d50e5-3944-4075-b07c-dc7b3890aaf9 is now active with session: db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2026-03-17T14:22:07.013670142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013682776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577,"session_id":"db256ae7-4d8d-4c4d-aa73-8eeab63cfcc4"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.01369612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} {"msg":"Activated sub_volume ff1d50e5-3944-4075-b07c-dc7b3890aaf9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:07.013707823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 296 len 28672 data: 9 7 8 10 10 8 8 02/30 Read block 192 len 8192 03/30 Read block 457 len 16384 04/30 Flush 05/30 Write block 41 len 40960 data: 7 9 8 8 8 8 11 12 11 10 06/30 Write block 503 len 12288 data: 9 10 12 07/30 Write block 294 len 36864 data: 10 9 10 8 9 11 11 9 9 08/30 Read block 234 len 40960 09/30 Read block 254 len 8192 10/30 Read block 395 len 20480 11/30 Write block 427 len 40960 data: 7 6 5 5 6 6 7 6 8 7 12/30 Read block 15 len 24576 13/30 Read block 185 len 24576 14/30 Flush 15/30 Write block 8 len 16384 data: 5 7 7 9 16/30 Read block 348 len 16384 17/30 Flush 18/30 Write block 584 len 12288 data: 7 7 8 19/30 Read block 397 len 24576 20/30 Write block 254 len 12288 data: 12 13 12 21/30 Read block 135 len 36864 22/30 Write block 555 len 28672 data: 7 8 8 7 6 8 9 23/30 Write block 53 len 32768 data: 8 7 7 6 6 5 4 4 24/30 Read block 83 len 28672 25/30 Read block 147 len 20480 26/30 Write block 110 len 20480 data: 9 8 7 9 8 27/30 Write block 218 len 36864 data: 6 6 7 8 9 8 8 8 7 28/30 Read block 591 len 8192 29/30 Read block 395 len 16384 30/30 Read block 486 len 20480 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 26 27 Done 25 26 0 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:07.026876707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1577} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 78 138 138 127 F F F <--- 1 020-039 80 80 80 130 130 130 F F F 2 040-059 84 84 82 137 137 134 T T F <--- 3 060-079 80 80 80 130 130 130 T T F <--- 4 080-099 82 82 82 134 134 134 F F F 5 100-119 79 79 79 129 129 129 T T F <--- 6 120-139 81 81 81 132 132 132 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 83 83 83 135 135 135 T T F <--- 11 220-239 82 82 82 134 134 134 T T F <--- 12 240-259 82 82 82 134 134 134 T T F <--- 13 260-279 82 82 82 134 134 134 F F F 14 280-299 84 84 84 137 137 136 F F F <--- 15 300-319 84 84 84 137 137 136 F F F <--- 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 82 82 82 134 134 134 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 84 84 82 137 137 134 F F F <--- 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 84 84 81 137 137 132 F F F <--- 26 520-539 81 81 81 132 132 132 F F F 27 540-559 83 83 83 135 135 135 T T F <--- 28 560-579 79 79 79 129 129 129 T T F <--- 29 580-599 80 80 80 131 131 131 T T F <--- Max gen: 84, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 80 80 80 130 130 130 F F F 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 134 134 134 F F F 5 100-119 85 85 85 139 139 139 F F F 6 120-139 81 81 81 132 132 132 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 78 78 78 128 128 128 F F F 10 200-219 85 85 85 139 139 139 F F F 11 220-239 85 85 85 139 139 139 F F F 12 240-259 85 85 85 139 139 139 F F F 13 260-279 82 82 82 134 134 134 F F F 14 280-299 84 84 84 137 137 137 F F F 15 300-319 84 84 84 137 137 137 F F F 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 82 82 82 134 134 134 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 84 84 84 137 137 137 F F F 26 520-539 81 81 81 132 132 132 F F F 27 540-559 85 85 85 139 139 139 F F F 28 560-579 85 85 85 139 139 139 F F F 29 580-599 85 85 85 139 139 139 F F F Max gen: 85, Max flush: 139 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.358419221Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370073644Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370383576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370419817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.37043197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370450186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"Crucible 751d1c6b-9d8b-426e-883a-a20cd4a4c654 has session id: 39865c5a-e5b9-4ced-b8a0-5a8b04c1a591","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370771672Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Upstairs opts: Upstairs UUID: 751d1c6b-9d8b-426e-883a-a20cd4a4c654, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370809853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Crucible stats registered with UUID: 751d1c6b-9d8b-426e-883a-a20cd4a4c654","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370825438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:21.370838792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.370990157Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371025947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371041702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371182694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371199989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"751d1c6b-9d8b-426e-883a-a20cd4a4c654 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371222736Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.37123592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371250464Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371264558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371278152Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371292317Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371306961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"ds_connection connected from Ok(127.0.0.1:57761)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371352775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"ds_connection connected from Ok(127.0.0.1:42834)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.37137141Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"ds_connection connected from Ok(127.0.0.1:34900)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.371510362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372181482Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.37221024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372450112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372776929Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[0]R flush_numbers[0..12]: [138, 130, 139, 139, 134, 139, 132, 126, 121, 128, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372800126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[0]R generation[0..12]: [84, 80, 85, 85, 82, 85, 81, 78, 74, 78, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372815951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372832596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[1]R flush_numbers[0..12]: [138, 130, 139, 139, 134, 139, 132, 126, 121, 128, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372859764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[1]R generation[0..12]: [84, 80, 85, 85, 82, 85, 81, 78, 74, 78, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372875338Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372891693Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[2]R flush_numbers[0..12]: [138, 130, 139, 139, 134, 139, 132, 126, 121, 128, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372907598Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[2]R generation[0..12]: [84, 80, 85, 85, 82, 85, 81, 78, 74, 78, 85, 85]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372926474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372942898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372956222Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372970607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.372989792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373003677Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"":"downstairs","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373017501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373030164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"0","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373044339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"1","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373058663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"client":"2","":"downstairs","session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373072847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373086301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"751d1c6b-9d8b-426e-883a-a20cd4a4c654 is now active with session: 39865c5a-e5b9-4ced-b8a0-5a8b04c1a591","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373101056Z","hostnameDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.37311723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586,"session_id":"39865c5a-e5b9-4ced-b8a0-5a8b04c1a591"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373130975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} {"msg":"Activated sub_volume 751d1c6b-9d8b-426e-883a-a20cd4a4c654","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:21.373142468Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 187 len 20480 data: 7 8 7 7 8 03/30 Flush 04/30 Write block 559 len 40960 data: 7 9 10 10 11 12 8 8 9 9 05/30 Flush 06/30 Write block 577 len 24576 data: 4 4 4 9 8 8 07/30 Read block 195 len 28672 08/30 Read block 494 len 12288 09/30 Flush 10/30 Write block 533 len 8192 data: 11 9 11/30 Read block 408 len 12288 12/30 Flush 13/30 Write block 35 len 20480 data: 7 7 8 8 8 14/30 Read block 2 len 16384 15/30 Read block 284 len 40960 16/30 Write block 124 len 24576 data: 5 4 5 5 6 6 17/30 Write block 277 len 36864 data: 6 7 7 8 7 6 7 8 9 18/30 Read block 372 len 16384 19/30 Write block 223 len 20480 data: 9 9 9 8 6 20/30 Read block 144 len 20480 21/30 Read block 322 len 36864 22/30 Read block 124 len 24576 23/30 Write block 267 len 12288 data: 12 9 10 24/30 Flush 25/30 Write block 186 len 4096 data: 6 26/30 Write block 556 len 12288 data: 9 9 8 27/30 Read block 70 len 8192 28/30 Read block 152 len 40960 29/30 Read block 126 len 16384 30/30 Read block 457 len 40960 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 27 0 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1002)) Some(JobId(1023)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:21.386033469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1586} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 86 80 86 145 130 145 F F F <--- 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 134 134 134 F F F 5 100-119 85 85 85 139 139 139 F F F 6 120-139 86 81 86 145 132 145 F F F <--- 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 86 86 86 141 141 141 T F T <--- 10 200-219 85 85 85 139 139 139 F F F 11 220-239 86 85 86 145 139 145 F F F <--- 12 240-259 85 85 85 139 139 139 F F F 13 260-279 86 82 86 145 134 145 F F F <--- 14 280-299 86 84 86 145 137 145 F F F <--- 15 300-319 84 84 84 137 137 137 F F F 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 82 82 82 134 134 134 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 84 84 84 137 137 137 F F F 26 520-539 86 81 86 144 132 144 F F F <--- 27 540-559 86 85 86 142 139 142 T F T <--- 28 560-579 86 85 86 143 139 143 F F F <--- 29 580-599 86 85 86 143 139 143 F F F <--- Max gen: 86, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 86 86 86 145 145 145 F F F 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 134 134 134 F F F 5 100-119 85 85 85 139 139 139 F F F 6 120-139 86 86 86 145 145 145 F F F 7 140-159 78 78 78 126 126 126 F F F 8 160-179 74 74 74 121 121 121 F F F 9 180-199 87 87 87 146 146 146 F F F 10 200-219 85 85 85 139 139 139 F F F 11 220-239 86 86 86 145 145 145 F F F 12 240-259 85 85 85 139 139 139 F F F 13 260-279 86 86 86 145 145 145 F F F 14 280-299 86 86 86 145 145 145 F F F 15 300-319 84 84 84 137 137 137 F F F 16 320-339 80 80 80 131 131 131 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 78 78 78 128 128 128 F F F 19 380-399 82 82 82 134 134 134 F F F 20 400-419 78 78 78 127 127 127 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 74 74 74 121 121 121 F F F 25 500-519 84 84 84 137 137 137 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 87 87 87 146 146 146 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 87, Max flush: 146 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.707300282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.718952284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719257455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719294256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719306219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719324085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"Crucible 79a0f305-f37a-4ed3-9df5-b21088d82ddc has session id: 581f1bcc-95f7-4450-af80-5b4c450b6b14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719680311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Upstairs opts: Upstairs UUID: 79a0f305-f37a-4ed3-9df5-b21088d82ddc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719715301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Crucible stats registered with UUID: 79a0f305-f37a-4ed3-9df5-b21088d82ddc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719731266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:35.71974508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719887963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.71991197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.719926584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720072598Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720089403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"79a0f305-f37a-4ed3-9df5-b21088d82ddc active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.72011266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720128474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720142628Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720156693Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720170617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720185021Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720199235Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:35181)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.72021423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:54155)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720229214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"ds_connection connected from Ok(127.0.0.1:56225)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720244179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720879698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.720914138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721263532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721641475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[0]R flush_numbers[0..12]: [138, 145, 139, 139, 134, 139, 145, 126, 121, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721667673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[0]R generation[0..12]: [84, 86, 85, 85, 82, 85, 86, 78, 74, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721684538Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721704384Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[1]R flush_numbers[0..12]: [138, 145, 139, 139, 134, 139, 145, 126, 121, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721720399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[1]R generation[0..12]: [84, 86, 85, 85, 82, 85, 86, 78, 74, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721735953Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721752588Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[2]R flush_numbers[0..12]: [138, 145, 139, 139, 134, 139, 145, 126, 121, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721768713Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[2]R generation[0..12]: [84, 86, 85, 85, 82, 85, 86, 78, 74, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721783897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721800242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721813656Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721834012Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721847566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.7218608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"":"downstairs","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721875215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721887928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"0","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721902133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"1","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721918258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"client":"2","":"downstairs","session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721932682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721949367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"79a0f305-f37a-4ed3-9df5-b21088d82ddc is now active with session: 581f1bcc-95f7-4450-af80-5b4c450b6b14","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721964381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721977305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595,"session_id":"581f1bcc-95f7-4450-af80-5b4c450b6b14"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.721990549Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} {"msg":"Activated sub_volume 79a0f305-f37a-4ed3-9df5-b21088d82ddc","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:35.722002133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 541 len 24576 data: 8 7 6 7 6 6 02/30 Read block 25 len 40960 03/30 Read block 211 len 28672 04/30 Write block 376 len 28672 data: 6 4 6 6 7 10 7 05/30 Read block 211 len 16384 06/30 Read block 401 len 24576 07/30 Read block 372 len 12288 08/30 Read block 458 len 20480 09/30 Write block 270 len 16384 data: 10 11 13 13 10/30 Write block 492 len 36864 data: 11 9 8 6 4 4 4 4 7 11/30 Read block 439 len 28672 12/30 Read block 210 len 28672 13/30 Read block 138 len 40960 14/30 Write block 389 len 4096 data: 4 15/30 Flush 16/30 Write block 89 len 36864 data: 8 6 5 7 8 6 6 6 8 17/30 Read block 86 len 16384 18/30 Write block 316 len 8192 data: 6 6 19/30 Read block 268 len 32768 20/30 Read block 488 len 32768 21/30 Read block 271 len 20480 22/30 Write block 158 len 28672 data: 6 6 4 3 3 4 4 23/30 Write block 106 len 20480 data: 9 9 6 8 10 24/30 Write block 287 len 16384 data: 9 10 10 10 25/30 Write block 407 len 32768 data: 5 6 5 4 5 6 5 8 26/30 Write block 321 len 8192 data: 8 7 27/30 Write block 135 len 20480 data: 9 7 8 7 5 28/30 Read block 531 len 32768 29/30 Read block 419 len 12288 30/30 Write block 494 len 28672 data: 9 7 5 5 5 5 8 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 10 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:35.734268557Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1595} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 86 86 86 145 145 145 F F F 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 82 82 82 134 134 134 T T F <--- 5 100-119 85 85 85 139 139 139 T T F <--- 6 120-139 86 86 86 145 145 145 T T F <--- 7 140-159 78 78 78 126 126 126 T T F <--- 8 160-179 74 74 74 121 121 121 T T F <--- 9 180-199 87 87 87 146 146 146 F F F 10 200-219 85 85 85 139 139 139 F F F 11 220-239 86 86 86 145 145 145 F F F 12 240-259 85 85 85 139 139 139 F F F 13 260-279 88 88 86 147 147 145 F F F <--- 14 280-299 86 86 86 145 145 145 T T F <--- 15 300-319 84 84 84 137 137 137 T T F <--- 16 320-339 80 80 80 131 131 131 T T F <--- 17 340-359 78 78 78 127 127 127 F F F 18 360-379 88 88 78 147 147 128 F F F <--- 19 380-399 88 88 82 147 147 134 F F F <--- 20 400-419 78 78 78 127 127 127 T T F <--- 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 88 88 74 147 147 121 T T F <--- 25 500-519 88 88 84 147 147 137 T T F <--- 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 87 147 147 146 F F T <--- 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 88, Max flush: 147 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 86 86 86 145 145 145 F F F 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 89 89 89 148 148 148 F F F 7 140-159 89 89 89 148 148 148 F F F 8 160-179 89 89 89 148 148 148 F F F 9 180-199 87 87 87 146 146 146 F F F 10 200-219 85 85 85 139 139 139 F F F 11 220-239 86 86 86 145 145 145 F F F 12 240-259 85 85 85 139 139 139 F F F 13 260-279 88 88 88 147 147 147 F F F 14 280-299 89 89 89 148 148 148 F F F 15 300-319 89 89 89 148 148 148 F F F 16 320-339 89 89 89 148 148 148 F F F 17 340-359 78 78 78 127 127 127 F F F 18 360-379 88 88 88 147 147 147 F F F 19 380-399 88 88 88 147 147 147 F F F 20 400-419 89 89 89 148 148 148 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 89 89 89 148 148 148 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 89, Max flush: 148 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.068613596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.080323385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.080655094Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.080692515Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.080704668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.080722764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"Crucible 8b5258c1-ceaa-4381-9300-f8d6a515b996 has session id: 952a5cd0-f86e-4848-b8f3-9e1910279c56","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081072158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Upstairs opts: Upstairs UUID: 8b5258c1-ceaa-4381-9300-f8d6a515b996, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081136327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Crucible stats registered with UUID: 8b5258c1-ceaa-4381-9300-f8d6a515b996","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081155042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:50.081168386Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081333016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081369436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081386351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081520261Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081536046Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"8b5258c1-ceaa-4381-9300-f8d6a515b996 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081551161Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081564805Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081578889Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081593493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081607667Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081621882Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081636436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"ds_connection connected from Ok(127.0.0.1:38749)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081759853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"ds_connection connected from Ok(127.0.0.1:43095)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081781879Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"ds_connection connected from Ok(127.0.0.1:45615)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.081800245Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082528772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082558571Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082601784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082961341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[0]R flush_numbers[0..12]: [138, 145, 139, 139, 148, 148, 148, 148, 148, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082983907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[0]R generation[0..12]: [84, 86, 85, 85, 89, 89, 89, 89, 89, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.082999812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083016657Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[1]R flush_numbers[0..12]: [138, 145, 139, 139, 148, 148, 148, 148, 148, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083032802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[1]R generation[0..12]: [84, 86, 85, 85, 89, 89, 89, 89, 89, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083048297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083064792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[2]R flush_numbers[0..12]: [138, 145, 139, 139, 148, 148, 148, 148, 148, 146, 139, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083080576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[2]R generation[0..12]: [84, 86, 85, 85, 89, 89, 89, 89, 89, 87, 85, 86]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083095571Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083111705Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.08312826Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:22:50.083142715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.08316019Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083173834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"":"downstairs","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083188358Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083201322Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"0","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083215346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"1","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.08322916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"client":"2","":"downstairs","session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083243535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"nameDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"crucible","level":30,"time":"2026-03-17T14:22:50.08326006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"8b5258c1-ceaa-4381-9300-f8d6a515b996 is now active with session: 952a5cd0-f86e-4848-b8f3-9e1910279c56","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083275594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083288908Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604,"session_id":"952a5cd0-f86e-4848-b8f3-9e1910279c56"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083302592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} {"msg":"Activated sub_volume 8b5258c1-ceaa-4381-9300-f8d6a515b996","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:22:50.083314266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 137 len 28672 data: 9 8 6 7 6 6 6 03/30 Write block 392 len 28672 data: 4 4 6 5 6 6 6 04/30 Read block 389 len 8192 05/30 Write block 360 len 28672 data: 7 8 6 6 5 5 4 06/30 Read block 234 len 16384 07/30 Read block 396 len 12288 08/30 Write block 239 len 20480 data: 14 12 12 8 7 09/30 Read block 252 len 8192 10/30 Write block 387 len 32768 data: 7 5 5 5 5 5 5 7 11/30 Read block 488 len 8192 12/30 Write block 27 len 8192 data: 6 5 13/30 Write block 316 len 16384 data: 7 7 7 8 14/30 Write block 339 len 40960 data: 7 6 8 9 11 11 10 14 10 11 15/30 Write block 323 len 12288 data: 7 7 8 16/30 Read block 254 len 12288 17/30 Read block 506 len 28672 18/30 Read block 84 len 8192 19/30 Read block 161 len 28672 20/30 Read block 422 len 4096 21/30 Write block 204 len 20480 data: 8 6 5 5 7 22/30 Read block 543 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 113 len 12288 data: 10 9 7 26/30 Read block 291 len 28672 27/30 Read block 175 len 24576 28/30 Write block 272 len 36864 data: 14 14 10 7 6 7 8 8 9 29/30 Write block 326 len 8192 data: 7 7 30/30 Write block 341 len 24576 data: 9 10 12 12 11 15 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 24 3 30 Done 26 5 26 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1000)) Some(JobId(1023)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:22:50.095194326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1604} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 90 86 90 150 145 150 F F F <--- 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 89 89 89 148 148 148 T F T <--- 6 120-139 90 89 90 150 148 150 F T F <--- 7 140-159 90 89 90 150 148 150 F T F <--- 8 160-179 89 89 89 148 148 148 F F F 9 180-199 87 87 87 146 146 146 F F F 10 200-219 90 85 90 150 139 150 F F F <--- 11 220-239 90 86 90 150 145 150 F F F <--- 12 240-259 90 85 90 150 139 150 F F F <--- 13 260-279 88 88 88 147 147 147 T F T <--- 14 280-299 89 89 89 148 148 148 T F T <--- 15 300-319 90 89 90 150 148 150 F F F <--- 16 320-339 90 89 90 150 148 150 T F T <--- 17 340-359 90 78 90 150 127 150 T F T <--- 18 360-379 90 88 90 150 147 150 F T F <--- 19 380-399 90 88 90 150 147 150 F T F <--- 20 400-419 89 89 89 148 148 148 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 89 89 89 148 148 148 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 90, Max flush: 150 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 138 138 138 F F F 1 020-039 90 90 90 150 150 150 F F F 2 040-059 85 85 85 139 139 139 F F F 3 060-079 85 85 85 139 139 139 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 90 90 90 150 150 150 F F F 7 140-159 90 90 90 150 150 150 F F F 8 160-179 89 89 89 148 148 148 F F F 9 180-199 87 87 87 146 146 146 F F F 10 200-219 90 90 90 150 150 150 F F F 11 220-239 90 90 90 150 150 150 F F F 12 240-259 90 90 90 150 150 150 F F F 13 260-279 91 91 91 151 151 151 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 91 91 91 151 151 151 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 90 90 90 150 150 150 F F F 19 380-399 90 90 90 150 150 150 F F F 20 400-419 89 89 89 148 148 148 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 89 89 89 148 148 148 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 91, Max flush: 151 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.4309295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.443263385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.443630674Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.443670606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.44368308Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.443701985Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"Crucible 61f87670-870c-453d-9d77-5513fddd9d54 has session id: c42646a1-ef63-4763-a59b-12fceeb3b65a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444126482Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Upstairs opts: Upstairs UUID: 61f87670-870c-453d-9d77-5513fddd9d54, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444162593Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Crucible stats registered with UUID: 61f87670-870c-453d-9d77-5513fddd9d54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444178767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:04.444192692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.44432041Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444344817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444359661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444511687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444529102Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"61f87670-870c-453d-9d77-5513fddd9d54 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444547067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444563272Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444577826Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444591991Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444605965Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444620339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444634893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45144)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444781877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51770)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444945076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48418)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.444972904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.445581015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.445618366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.445635922Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.44603378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[0]R flush_numbers[0..12]: [138, 150, 139, 139, 148, 151, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446059338Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[0]R generation[0..12]: [84, 90, 85, 85, 89, 91, 90, 90, 89, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446075853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446096239Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[1]R flush_numbers[0..12]: [138, 150, 139, 139, 148, 151, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446112684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[1]R generation[0..12]: [84, 90, 85, 85, 89, 91, 90, 90, 89, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446128618Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446145393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[2]R flush_numbers[0..12]: [138, 150, 139, 139, 148, 151, 150, 150, 148, 146, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446161858Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[2]R generation[0..12]: [84, 90, 85, 85, 89, 91, 90, 90, 89, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446177383Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446194148Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446208112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:23:04.446222736Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446241582Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446255606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"":"downstairs","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.44626986Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446282744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"0","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446297239Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"1","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446311663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"client":"2","":"downstairs","session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446328048Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446344923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"61f87670-870c-453d-9d77-5513fddd9d54 is now active with session: c42646a1-ef63-4763-a59b-12fceeb3b65a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446360447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446373531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613,"session_id":"c42646a1-ef63-4763-a59b-12fceeb3b65a"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446387396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} {"msg":"Activated sub_volume 61f87670-870c-453d-9d77-5513fddd9d54","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:04.446399059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 57 len 36864 data: 7 6 5 5 5 6 6 6 6 03/30 Read block 569 len 8192 04/30 Write block 196 len 8192 data: 10 12 05/30 Read block 262 len 20480 06/30 Read block 383 len 12288 07/30 Write block 19 len 12288 data: 6 3 4 08/30 Read block 432 len 40960 09/30 Read block 128 len 8192 10/30 Write block 196 len 36864 data: 11 13 12 11 11 9 10 9 9 11/30 Write block 337 len 4096 data: 5 12/30 Read block 531 len 16384 13/30 Read block 516 len 36864 14/30 Read block 534 len 32768 15/30 Write block 163 len 24576 data: 5 5 5 4 4 4 16/30 Read block 494 len 20480 17/30 Read block 181 len 36864 18/30 Read block 30 len 20480 19/30 Read block 313 len 24576 20/30 Read block 163 len 4096 21/30 Read block 67 len 24576 22/30 Write block 392 len 4096 data: 6 23/30 Flush 24/30 Write block 183 len 4096 data: 6 25/30 Flush 26/30 Write block 401 len 40960 data: 9 10 8 6 6 5 6 7 6 5 27/30 Write block 221 len 8192 data: 9 10 28/30 Write block 212 len 40960 data: 5 5 5 5 7 6 7 7 8 10 29/30 Read block 411 len 40960 30/30 Read block 158 len 40960 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 29 3 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1000)) Some(JobId(1024)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:04.458861192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1613} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 84 92 153 138 153 F F F <--- 1 020-039 92 90 92 153 150 153 F F F <--- 2 040-059 92 85 92 153 139 153 F T F <--- 3 060-079 92 85 92 153 139 153 F T F <--- 4 080-099 89 89 89 148 148 148 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 90 90 90 150 150 150 F F F 7 140-159 90 90 90 150 150 150 F F F 8 160-179 92 89 92 153 148 153 F F F <--- 9 180-199 92 87 92 154 146 154 F T F <--- 10 200-219 92 90 92 153 150 153 T F T <--- 11 220-239 90 90 90 150 150 150 T F T <--- 12 240-259 90 90 90 150 150 150 F F F 13 260-279 91 91 91 151 151 151 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 91 92 153 151 153 F F F <--- 17 340-359 91 91 91 151 151 151 F F F 18 360-379 90 90 90 150 150 150 F F F 19 380-399 92 90 92 153 150 153 F F F <--- 20 400-419 89 89 89 148 148 148 T F T <--- 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 89 89 89 148 148 148 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 92, Max flush: 154 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 153 153 153 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 92 92 92 153 153 153 F F F 3 060-079 92 92 92 153 153 153 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 90 90 90 150 150 150 F F F 7 140-159 90 90 90 150 150 150 F F F 8 160-179 92 92 92 153 153 153 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 93 93 93 155 155 155 F F F 11 220-239 93 93 93 155 155 155 F F F 12 240-259 90 90 90 150 150 150 F F F 13 260-279 91 91 91 151 151 151 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 92 92 153 153 153 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 90 90 90 150 150 150 F F F 19 380-399 92 92 92 153 153 153 F F F 20 400-419 93 93 93 155 155 155 F F F 21 420-439 84 84 84 137 137 137 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 89 89 89 148 148 148 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 88 88 88 147 147 147 F F F 28 560-579 86 86 86 143 143 143 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 93, Max flush: 155 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.811110986Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.822802499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823145241Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823196827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823209691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823227596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"Crucible 5b1cdaa1-0945-44a8-ac41-0b03ab701274 has session id: 2684e836-b2c5-40b7-b5c2-e43a2d1c91a4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823575079Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Upstairs opts: Upstairs UUID: 5b1cdaa1-0945-44a8-ac41-0b03ab701274, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.82361025Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Crucible stats registered with UUID: 5b1cdaa1-0945-44a8-ac41-0b03ab701274","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823626015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:18.82364297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823790574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823824044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.823839718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824002177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824026174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"5b1cdaa1-0945-44a8-ac41-0b03ab701274 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824038638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824051841Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824066346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.82408045Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824094684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824109179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824123523Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43942)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824166015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49146)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824339567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37470)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824367716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.82482017Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.824860983Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825231473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825576896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[0]R flush_numbers[0..12]: [153, 153, 153, 153, 148, 151, 150, 150, 153, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825601973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[0]R generation[0..12]: [92, 92, 92, 92, 89, 91, 90, 90, 92, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825617798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825634603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[1]R flush_numbers[0..12]: [153, 153, 153, 153, 148, 151, 150, 150, 153, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825650118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[1]R generation[0..12]: [92, 92, 92, 92, 89, 91, 90, 90, 92, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825665682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825682167Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[2]R flush_numbers[0..12]: [153, 153, 153, 153, 148, 151, 150, 150, 153, 154, 155, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825698112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[2]R generation[0..12]: [92, 92, 92, 92, 89, 91, 90, 90, 92, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825713447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825730081Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825746907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:23:18.825761141Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825779286Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.82579296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"":"downstairs","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825806874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825819748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"0","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825834082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"1","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825848067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"client":"2","":"downstairs","session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825862441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825875635Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"5b1cdaa1-0945-44a8-ac41-0b03ab701274 is now active with session: 2684e836-b2c5-40b7-b5c2-e43a2d1c91a4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.82589345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825906544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622,"session_id":"2684e836-b2c5-40b7-b5c2-e43a2d1c91a4"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825920898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} {"msg":"Activated sub_volume 5b1cdaa1-0945-44a8-ac41-0b03ab701274","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:18.825932492Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 422 len 28672 data: 7 8 8 8 7 8 7 03/30 Read block 201 len 12288 04/30 Write block 558 len 4096 data: 9 05/30 Write block 216 len 16384 data: 8 7 8 8 06/30 Read block 261 len 28672 07/30 Read block 55 len 4096 08/30 Read block 254 len 4096 09/30 Read block 330 len 4096 10/30 Flush 11/30 Flush 12/30 Write block 553 len 40960 data: 7 9 8 10 10 10 8 10 11 11 13/30 Read block 123 len 28672 14/30 Read block 485 len 32768 15/30 Write block 103 len 20480 data: 11 11 12 10 10 16/30 Write block 510 len 28672 data: 11 11 10 11 10 9 9 17/30 Read block 320 len 20480 18/30 Write block 54 len 40960 data: 8 8 7 8 7 6 6 6 7 7 19/30 Read block 135 len 36864 20/30 Read block 119 len 40960 21/30 Read block 70 len 16384 22/30 Read block 216 len 4096 23/30 Read block 171 len 20480 24/30 Read block 563 len 20480 25/30 Write block 268 len 40960 data: 10 11 11 12 15 15 11 8 7 8 26/30 Flush 27/30 Write block 230 len 32768 data: 5 7 9 11 11 13 11 13 28/30 Write block 222 len 28672 data: 11 10 10 10 9 7 5 29/30 Read block 509 len 4096 30/30 Read block 411 len 16384 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 7 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:18.838659534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1622} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 153 153 153 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 94 94 92 159 159 153 F F F <--- 3 060-079 94 94 92 159 159 153 F F F <--- 4 080-099 89 89 89 148 148 148 F F F 5 100-119 94 94 91 159 159 151 F F F <--- 6 120-139 90 90 90 150 150 150 F F F 7 140-159 90 90 90 150 150 150 F F F 8 160-179 92 92 92 153 153 153 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 94 94 93 157 157 155 F F F <--- 11 220-239 93 93 93 155 155 155 T T F <--- 12 240-259 90 90 90 150 150 150 F F F 13 260-279 94 94 91 159 159 151 F F F <--- 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 92 92 153 153 153 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 90 90 90 150 150 150 F F F 19 380-399 92 92 92 153 153 153 F F F 20 400-419 93 93 93 155 155 155 F F F 21 420-439 94 94 84 157 157 137 F F F <--- 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 89 159 159 148 F F F <--- 26 520-539 86 86 86 144 144 144 F F F 27 540-559 94 94 88 159 159 147 F F F <--- 28 560-579 94 94 86 159 159 143 F F F <--- 29 580-599 86 86 86 143 143 143 F F F Max gen: 94, Max flush: 159 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 153 153 153 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 94 94 94 159 159 159 F F F 3 060-079 94 94 94 159 159 159 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 94 94 94 159 159 159 F F F 6 120-139 90 90 90 150 150 150 F F F 7 140-159 90 90 90 150 150 150 F F F 8 160-179 92 92 92 153 153 153 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 94 94 94 157 157 157 F F F 11 220-239 95 95 95 160 160 160 F F F 12 240-259 90 90 90 150 150 150 F F F 13 260-279 94 94 94 159 159 159 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 92 92 153 153 153 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 90 90 90 150 150 150 F F F 19 380-399 92 92 92 153 153 153 F F F 20 400-419 93 93 93 155 155 155 F F F 21 420-439 94 94 94 157 157 157 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 94 94 94 159 159 159 F F F 28 560-579 94 94 94 159 159 159 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 95, Max flush: 160 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.158689937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:33.169444442Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.169491876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.179972559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180262745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180300446Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18031248Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180330925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"Crucible 3c92cc1f-30eb-413d-8938-c92f3bfbea2a has session id: 0790297f-915d-4970-8669-f07f8f7b5034","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180691333Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Upstairs opts: Upstairs UUID: 3c92cc1f-30eb-413d-8938-c92f3bfbea2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180727574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Crucible stats registered with UUID: 3c92cc1f-30eb-413d-8938-c92f3bfbea2a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180743618Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:33.180768886Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180922802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180947719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.180962334Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18111895Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181133645Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"3c92cc1f-30eb-413d-8938-c92f3bfbea2a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18115124Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181166675Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181180819Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181194723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181208227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181222671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181237246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"ds_connection connected from Ok(127.0.0.1:48603)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181290912Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"ds_connection connected from Ok(127.0.0.1:43584)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181306966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"ds_connection connected from Ok(127.0.0.1:32811)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.181411227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182069954Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182108745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18212578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182503173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[0]R flush_numbers[0..12]: [153, 153, 159, 159, 148, 159, 150, 150, 153, 154, 157, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18252739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[0]R generation[0..12]: [92, 92, 94, 94, 89, 94, 90, 90, 92, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182543085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.1825626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[1]R flush_numbers[0..12]: [153, 153, 159, 159, 148, 159, 150, 150, 153, 154, 157, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182579205Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[1]R generation[0..12]: [92, 92, 94, 94, 89, 94, 90, 90, 92, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18259448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182610905Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[2]R flush_numbers[0..12]: [153, 153, 159, 159, 148, 159, 150, 150, 153, 154, 157, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18262687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[2]R generation[0..12]: [92, 92, 94, 94, 89, 94, 90, 90, 92, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182642084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182658689Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182672363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182686767Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182706783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182720598Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"":"downstairs","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182734782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182747586Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"0","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.18276166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"1","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182777755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"client":"2","":"downstairs","session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182792019Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Set Downstairs and Upstairs active after reconciliationDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182808244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"3c92cc1f-30eb-413d-8938-c92f3bfbea2a is now active with session: 0790297f-915d-4970-8669-f07f8f7b5034","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182823738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182836592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631,"session_id":"0790297f-915d-4970-8669-f07f8f7b5034"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182850056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} {"msg":"Activated sub_volume 3c92cc1f-30eb-413d-8938-c92f3bfbea2a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:33.182861369Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 36864 data: 7 7 8 10 11 9 7 7 6 02/30 Read block 499 len 24576 03/30 Read block 439 len 20480 04/30 Read block 50 len 40960 05/30 Read block 242 len 40960 06/30 Flush 07/30 Write block 73 len 8192 data: 5 6 08/30 Write block 352 len 36864 data: 10 12 11 12 11 9 7 5 8 09/30 Write block 558 len 20480 data: 11 9 11 12 12 10/30 Write block 569 len 4096 data: 9 11/30 Read block 116 len 20480 12/30 Write block 361 len 20480 data: 9 7 7 6 6 13/30 Flush 14/30 Write block 119 len 40960 data: 3 3 5 6 5 6 5 6 6 7 15/30 Read block 152 len 20480 16/30 Flush 17/30 Write block 273 len 12288 data: 16 12 9 18/30 Read block 60 len 28672 19/30 Read block 103 len 12288 20/30 Write block 132 len 24576 data: 7 7 7 10 8 10 21/30 Write block 234 len 36864 data: 12 14 12 14 14 15 13 13 9 22/30 Write block 431 len 36864 data: 7 7 8 7 9 8 6 4 4 23/30 Write block 154 len 32768 data: 7 7 7 7 7 7 5 4 24/30 Read block 332 len 32768 25/30 Read block 551 len 28672 26/30 Flush 27/30 Write block 542 len 24576 data: 8 7 8 7 7 7 28/30 Read block 191 len 40960 29/30 Read block 515 len 36864 30/30 Read block 330 len 20480 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 0 30 Done 1 29 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:33.196428182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1631} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 153 153 153 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 94 94 94 159 159 159 F F F 3 060-079 94 96 96 159 162 162 F F F <--- 4 080-099 89 89 89 148 148 148 F F F 5 100-119 94 96 96 159 163 163 F F F <--- 6 120-139 90 96 96 150 164 164 F F F <--- 7 140-159 90 96 96 150 164 164 F F F <--- 8 160-179 92 96 96 153 164 164 F F F <--- 9 180-199 92 92 92 154 154 154 F F F 10 200-219 94 94 94 157 157 157 F F F 11 220-239 95 96 96 160 164 164 F F F <--- 12 240-259 90 96 96 150 164 164 F F F <--- 13 260-279 94 96 96 159 164 164 F F F <--- 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 92 92 153 153 153 F F F 17 340-359 91 96 96 151 162 162 F F F <--- 18 360-379 90 96 96 150 162 162 F F F <--- 19 380-399 92 96 96 153 161 161 T F F <--- 20 400-419 93 96 96 155 161 161 T F F <--- 21 420-439 94 96 96 157 164 164 F F F <--- 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 94 96 96 159 162 162 F T T <--- 28 560-579 94 96 96 159 162 162 F F F <--- 29 580-599 86 86 86 143 143 143 F F F Max gen: 96, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 153 153 153 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 94 94 94 159 159 159 F F F 3 060-079 96 96 96 162 162 162 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 96 96 96 163 163 163 F F F 6 120-139 96 96 96 164 164 164 F F F 7 140-159 96 96 96 164 164 164 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 94 94 94 157 157 157 F F F 11 220-239 96 96 96 164 164 164 F F F 12 240-259 96 96 96 164 164 164 F F F 13 260-279 96 96 96 164 164 164 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 92 92 92 153 153 153 F F F 17 340-359 96 96 96 162 162 162 F F F 18 360-379 96 96 96 162 162 162 F F F 19 380-399 96 96 96 161 161 161 F F F 20 400-419 96 96 96 161 161 161 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 80 80 80 130 130 130 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 86 86 86 144 144 144 F F F 27 540-559 97 97 97 165 165 165 F F F 28 560-579 96 96 96 162 162 162 F F F 29 580-599 86 86 86 143 143 143 F F F Max gen: 97, Max flush: 165 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.527548941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539237194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539533832Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539569123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539581166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539598702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"Crucible 597eb2ac-1bc1-4950-8929-49cfb00b0fb2 has session id: 40e24eb7-1f94-4e80-9e43-8739bfca5670","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539945715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Upstairs opts: Upstairs UUID: 597eb2ac-1bc1-4950-8929-49cfb00b0fb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539979985Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Crucible stats registered with UUID: 597eb2ac-1bc1-4950-8929-49cfb00b0fb2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.539998771Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:47.540012545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540166061Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540190228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540205392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540357308Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540374733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"597eb2ac-1bc1-4950-8929-49cfb00b0fb2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540396699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540409613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540423687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540437812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540451686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.54046604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540480194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"ds_connection connected from Ok(127.0.0.1:51793)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540686656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"ds_connection connected from Ok(127.0.0.1:35935)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540714624Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"ds_connection connected from Ok(127.0.0.1:47607)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.540749034Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.541279462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.541329477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.541705139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542051412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[0]R flush_numbers[0..12]: [153, 153, 159, 162, 148, 163, 164, 164, 164, 154, 157, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.54207562Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[0]R generation[0..12]: [92, 92, 94, 96, 89, 96, 96, 96, 96, 92, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542091754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542108409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[1]R flush_numbers[0..12]: [153, 153, 159, 162, 148, 163, 164, 164, 164, 154, 157, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542124524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[1]R generation[0..12]: [92, 92, 94, 96, 89, 96, 96, 96, 96, 92, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542140279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542156874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[2]R flush_numbers[0..12]: [153, 153, 159, 162, 148, 163, 164, 164, 164, 154, 157, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542172498Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[2]R generation[0..12]: [92, 92, 94, 96, 89, 96, 96, 96, 96, 92, 94, 96]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542187673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542207479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542221283Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542240589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542254763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542267937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"":"downstairs","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542282131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542294825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"0","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542308949Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"1","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542323273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"client":"2","":"downstairs","session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542337588Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542353712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"597eb2ac-1bc1-4950-8929-49cfb00b0fb2 is now active with session: 40e24eb7-1f94-4e80-9e43-8739bfca5670","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542368747Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542381331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640,"session_id":"40e24eb7-1f94-4e80-9e43-8739bfca5670"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542394805Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} {"msg":"Activated sub_volume 597eb2ac-1bc1-4950-8929-49cfb00b0fb2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:23:47.542406368Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 32768 data: 9 7 8 6 6 6 6 7 02/30 Write block 237 len 36864 data: 15 15 16 14 14 10 8 7 5 03/30 Read block 473 len 4096 04/30 Read block 508 len 28672 05/30 Read block 95 len 28672 06/30 Write block 14 len 4096 data: 8 07/30 Write block 590 len 24576 data: 7 7 6 7 5 4 08/30 Write block 446 len 32768 data: 9 3 4 5 8 8 8 7 09/30 Write block 258 len 16384 data: 9 8 8 7 10/30 Write block 362 len 12288 data: 8 8 7 11/30 Read block 101 len 12288 12/30 Write block 314 len 36864 data: 4 6 8 8 8 9 9 9 8 13/30 Read block 142 len 8192 14/30 Write block 339 len 8192 data: 8 7 15/30 Write block 262 len 32768 data: 7 6 6 6 9 13 11 12 16/30 Read block 556 len 32768 17/30 Flush 18/30 Write block 533 len 32768 data: 12 10 8 8 8 7 7 6 19/30 Write block 56 len 24576 data: 8 9 8 7 7 7 20/30 Read block 61 len 20480 21/30 Read block 105 len 12288 22/30 Flush 23/30 Write block 442 len 28672 data: 4 6 10 11 10 4 5 24/30 Write block 112 len 32768 data: 8 11 10 8 5 6 4 4 25/30 Write block 136 len 12288 data: 9 11 9 26/30 Read block 336 len 8192 27/30 Write block 113 len 40960 data: 12 11 9 6 7 5 5 4 6 7 28/30 Write block 195 len 36864 data: 6 12 14 13 12 12 10 11 10 29/30 Write block 272 len 8192 data: 16 17 30/30 Write block 538 len 36864 data: 8 8 7 9 9 8 9 8 8 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 9 Done Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 30 35 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:23:47.554788267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 92 166 166 153 F F F <--- 1 020-039 92 92 92 153 153 153 F F F 2 040-059 98 98 94 167 167 159 F F F <--- 3 060-079 98 98 96 167 167 162 F F F <--- 4 080-099 89 89 89 148 148 148 F F F 5 100-119 96 96 96 163 163 163 T T F <--- 6 120-139 96 96 96 164 164 164 T T F <--- 7 140-159 96 96 96 164 164 164 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 92 92 92 154 154 154 T T F <--- 10 200-219 94 94 94 157 157 157 T T F <--- 11 220-239 98 98 96 166 166 164 F F F <--- 12 240-259 98 98 96 166 166 164 F F F <--- 13 260-279 98 98 96 166 166 164 T T F <--- 14 280-299 91 91 91 151 151 151 F F F 15 300-319 98 98 90 166 166 150 F F F <--- 16 320-339 98 98 92 166 166 153 F F F <--- 17 340-359 98 98 96 166 166 162 F F F <--- 18 360-379 98 98 96 166 166 162 F F F <--- 19 380-399 98 98 96 166 166 161 F F F <--- 20 400-419 96 96 96 161 161 161 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 98 98 80 166 166 130 T T F <--- 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 98 98 86 167 167 144 T T F <--- 27 540-559 98 98 97 167 167 165 T T F <--- 28 560-579 96 96 96 162 162 162 F F F 29 580-599 98 98 86 166 166 143 F F F <--- Max gen: 98, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 166 166 166 F F F 1 020-039 92 92 92 153 153 153 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 98 98 98 167 167 167 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 99 99 99 168 168 168 F F F 6 120-139 99 99 99 168 168 168 F F F 7 140-159 96 96 96 164 164 164 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 99 99 99 168 168 168 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 98 98 98 166 166 166 F F F 13 260-279 99 99 99 168 168 168 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 98 98 98 166 166 166 F F F 18 360-379 98 98 98 166 166 166 F F F 19 380-399 98 98 98 166 166 166 F F F 20 400-419 96 96 96 161 161 161 F F F 21 420-439 96 96 96 164 164 164 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 99 99 99 168 168 168 F F F 27 540-559 99 99 99 168 168 168 F F F 28 560-579 96 96 96 162 162 162 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 99, Max flush: 168 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.900401963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.912127537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.91244059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.912478571Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.912490305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.9125082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"Crucible 43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2 has session id: c8747c2e-77ab-45dc-af2a-2280a086c685","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.912860095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Upstairs opts: Upstairs UUID: 43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.912894725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Crucible stats registered with UUID: 43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.91291065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:01.912924304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913101407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913128385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913143179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913271397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913287982Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913305458Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913318181Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913332366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.91334602Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913359924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913374308Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913388172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"ds_connection connected from Ok(127.0.0.1:32937)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913572247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"ds_connection connected from Ok(127.0.0.1:61613)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913595924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"ds_connection connected from Ok(127.0.0.1:49111)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.913611069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.914280578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.914338916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.914699973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915070744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[0]R flush_numbers[0..12]: [166, 153, 167, 167, 148, 168, 168, 164, 164, 168, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915096281Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[0]R generation[0..12]: [98, 92, 98, 98, 89, 99, 99, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915115457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915132202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[1]R flush_numbers[0..12]: [166, 153, 167, 167, 148, 168, 168, 164, 164, 168, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915147827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[1]R generation[0..12]: [98, 92, 98, 98, 89, 99, 99, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915163361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915179806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[2]R flush_numbers[0..12]: [166, 153, 167, 167, 148, 168, 168, 164, 164, 168, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915195641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[2]R generation[0..12]: [98, 92, 98, 98, 89, 99, 99, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915210855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.91522722Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915240854Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915255729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P"Wait for a query_work_queue command to finish before sending IO ,"pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915275335Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915288629Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"":"downstairs","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915302703Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915315226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"0","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915331631Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"1","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915345635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"client":"2","":"downstairs","session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.91536118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:24:01.915374664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2 is now active with session: c8747c2e-77ab-45dc-af2a-2280a086c685","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915391189Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915404583Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649,"session_id":"c8747c2e-77ab-45dc-af2a-2280a086c685"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915418327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} {"msg":"Activated sub_volume 43e4607d-5bc4-4ce6-a0fa-b204e6eba8f2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:01.915429981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 593 len 28672 data: 8 6 5 5 4 4 3 02/30 Read block 76 len 28672 03/30 Read block 281 len 32768 04/30 Read block 296 len 36864 05/30 Write block 380 len 40960 data: 8 11 8 5 4 10 8 9 7 7 06/30 Flush 07/30 Write block 66 len 36864 data: 6 7 10 9 8 6 5 6 7 08/30 Write block 547 len 40960 data: 8 8 7 7 7 7 8 10 9 11 09/30 Write block 566 len 12288 data: 9 10 10 10/30 Write block 531 len 36864 data: 10 10 13 11 9 9 9 9 9 11/30 Write block 249 len 4096 data: 10 12/30 Write block 514 len 24576 data: 11 10 10 11 9 7 13/30 Write block 390 len 8192 data: 7 7 14/30 Read block 14 len 8192 15/30 Read block 203 len 12288 16/30 Read block 185 len 28672 17/30 Read block 582 len 16384 18/30 Write block 503 len 24576 data: 10 11 13 12 13 11 19/30 Write block 108 len 32768 data: 7 9 11 9 9 13 12 10 20/30 Flush 21/30 Write block 436 len 12288 data: 9 7 5 22/30 Read block 147 len 8192 23/30 Flush 24/30 Write block 189 len 24576 data: 8 8 9 7 6 6 25/30 Write block 65 len 36864 data: 7 7 8 11 10 9 7 6 7 26/30 Write block 17 len 20480 data: 8 7 7 4 5 27/30 Write block 246 len 20480 data: 6 6 9 11 12 28/30 Read block 73 len 4096 29/30 Write block 16 len 36864 data: 9 9 8 8 5 6 5 7 6 30/30 Read block 344 len 28672 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:01.928069957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1649} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 166 166 166 T F T <--- 1 020-039 92 92 92 153 153 153 T F T <--- 2 040-059 98 98 98 167 167 167 F F F 3 060-079 100 98 100 170 167 170 T F T <--- 4 080-099 89 89 89 148 148 148 F F F 5 100-119 100 99 100 170 168 170 F F F <--- 6 120-139 99 99 99 168 168 168 F F F 7 140-159 96 96 96 164 164 164 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 99 99 99 168 168 168 T F T <--- 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 100 98 100 170 166 170 T F T <--- 13 260-279 99 99 99 168 168 168 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 98 98 98 166 166 166 F F F 18 360-379 98 98 98 166 166 166 F F F 19 380-399 100 98 100 170 166 170 F F F <--- 20 400-419 96 96 96 161 161 161 F F F 21 420-439 100 96 100 171 164 171 F F F <--- 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 100 94 100 170 159 170 F F F <--- 26 520-539 100 99 100 170 168 170 F F F <--- 27 540-559 100 99 100 170 168 170 F F F <--- 28 560-579 100 96 100 170 162 170 F F F <--- 29 580-599 100 98 100 169 166 169 F T F <--- Max gen: 100, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 F F F 1 020-039 101 101 101 172 172 172 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 101 101 101 172 172 172 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 100 100 100 170 170 170 F F F 6 120-139 99 99 99 168 168 168 F F F 7 140-159 96 96 96 164 164 164 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 101 101 101 172 172 172 F F F 13 260-279 99 99 99 168 168 168 F F F 14 280-299 91 91 91 151 151 151 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 98 98 98 166 166 166 F F F 18 360-379 98 98 98 166 166 166 F F F 19 380-399 100 100 100 170 170 170 F F F 20 400-419 96 96 96 161 161 161 F F F 21 420-439 100 100 100 171 171 171 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 100 100 100 170 170 170 F F F 26 520-539 100 100 100 170 170 170 F F F 27 540-559 100 100 100 170 170 170 F F F 28 560-579 100 100 100 170 170 170 F F F 29 580-599 100 100 100 169 169 169 F F F Max gen: 101, Max flush: 172 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.256092501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:16.266855108Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.266911295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.277383385Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.277666139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.277704081Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.277715964Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.277733979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"Crucible e99ec8b4-3cf8-4525-bf5f-db022832dc2a has session id: 9e921da2-c3ba-45cd-b2b9-57ccbe40230a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278097608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Upstairs opts: Upstairs UUID: e99ec8b4-3cf8-4525-bf5f-db022832dc2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278133198Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Crucible stats registered with UUID: e99ec8b4-3cf8-4525-bf5f-db022832dc2a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278148913Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:16.278162357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278310351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278338219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278353054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278486654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278501708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"e99ec8b4-3cf8-4525-bf5f-db022832dc2a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278531957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278546392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278560696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.27857424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278588224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278602668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278618913Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52431)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278634058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52461)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278649062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46084)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.278733767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279431655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279468826Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279486011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279867715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[0]R flush_numbers[0..12]: [172, 172, 167, 172, 148, 170, 168, 164, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279892193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[0]R generation[0..12]: [101, 101, 98, 101, 89, 100, 99, 96, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279907987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279924592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[1]R flush_numbers[0..12]: [172, 172, 167, 172, 148, 170, 168, 164, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279940757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[1]R generation[0..12]: [101, 101, 98, 101, 89, 100, 99, 96, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279956252Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279972707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[2]R flush_numbers[0..12]: [172, 172, 167, 172, 148, 170, 168, 164, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.279992082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[2]R generation[0..12]: [101, 101, 98, 101, 89, 100, 99, 96, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280007617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280023782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280037426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.28005183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280065214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.28008434Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"":"downstairs","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280098594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280111778Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"0","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280126162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"1","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280140376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"client":"2","":"downstairs","session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280154811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:24:16.280168465Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"e99ec8b4-3cf8-4525-bf5f-db022832dc2a is now active with session: 9e921da2-c3ba-45cd-b2b9-57ccbe40230a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.28018494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280197754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658,"session_id":"9e921da2-c3ba-45cd-b2b9-57ccbe40230a"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280212898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} {"msg":"Activated sub_volume e99ec8b4-3cf8-4525-bf5f-db022832dc2a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:16.280224672Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 28672 data: 8 7 7 5 3 4 5 02/30 Write block 350 len 24576 data: 12 9 11 13 12 13 03/30 Write block 142 len 16384 data: 7 7 6 5 04/30 Read block 189 len 40960 05/30 Write block 148 len 20480 data: 7 8 8 6 7 06/30 Write block 98 len 20480 data: 7 7 10 9 12 07/30 Flush 08/30 Write block 590 len 16384 data: 8 8 7 9 09/30 Write block 375 len 8192 data: 6 7 10/30 Write block 107 len 4096 data: 11 11/30 Read block 174 len 24576 12/30 Flush 13/30 Write block 577 len 28672 data: 5 5 5 10 9 9 9 14/30 Read block 311 len 36864 15/30 Read block 90 len 24576 16/30 Write block 312 len 24576 data: 4 5 6 7 9 9 17/30 Write block 373 len 12288 data: 9 8 7 18/30 Flush 19/30 Write block 352 len 16384 data: 12 14 13 14 20/30 Read block 567 len 24576 21/30 Read block 264 len 36864 22/30 Read block 493 len 12288 23/30 Write block 280 len 4096 data: 10 24/30 Read block 39 len 36864 25/30 Read block 290 len 16384 26/30 Write block 392 len 12288 data: 8 6 8 27/30 Write block 339 len 24576 data: 9 8 10 11 13 13 28/30 Read block 166 len 24576 29/30 Write block 538 len 16384 data: 10 10 8 10 30/30 Read block 526 len 12288 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 1 26 Done 5 30 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:16.292155166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1658} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 F F F 1 020-039 101 101 101 172 172 172 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 101 101 101 172 172 172 F F F 4 080-099 89 102 102 148 173 173 F F F <--- 5 100-119 100 102 102 170 174 174 F F F <--- 6 120-139 99 99 99 168 168 168 F F F 7 140-159 96 102 102 164 173 173 T F F <--- 8 160-179 96 96 96 164 164 164 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 101 101 101 172 172 172 F F F 13 260-279 99 99 99 168 168 168 F F F 14 280-299 91 91 91 151 151 151 F T T <--- 15 300-319 98 102 102 166 175 175 T F F <--- 16 320-339 98 98 98 166 166 166 F T T <--- 17 340-359 98 102 102 166 173 173 T T T <--- 18 360-379 98 102 102 166 175 175 F F F <--- 19 380-399 100 100 100 170 170 170 F T T <--- 20 400-419 96 96 96 161 161 161 F F F 21 420-439 100 100 100 171 171 171 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 100 100 100 170 170 170 F F F 26 520-539 100 100 100 170 170 170 F T T <--- 27 540-559 100 100 100 170 170 170 F T T <--- 28 560-579 100 102 102 170 175 175 F F F <--- 29 580-599 100 102 102 169 175 175 F F F <--- Max gen: 102, Max flush: 175 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 F F F 1 020-039 101 101 101 172 172 172 F F F 2 040-059 98 98 98 167 167 167 F F F 3 060-079 101 101 101 172 172 172 F F F 4 080-099 102 102 102 173 173 173 F F F 5 100-119 102 102 102 174 174 174 F F F 6 120-139 99 99 99 168 168 168 F F F 7 140-159 102 102 102 173 173 173 F F F 8 160-179 96 96 96 164 164 164 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 101 101 101 172 172 172 F F F 13 260-279 99 99 99 168 168 168 F F F 14 280-299 103 103 103 176 176 176 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 103 103 103 176 176 176 F F F 18 360-379 102 102 102 175 175 175 F F F 19 380-399 103 103 103 176 176 176 F F F 20 400-419 96 96 96 161 161 161 F F F 21 420-439 100 100 100 171 171 171 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 89 89 148 148 148 F F F 25 500-519 100 100 100 170 170 170 F F F 26 520-539 103 103 103 176 176 176 F F F 27 540-559 103 103 103 176 176 176 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 102 102 102 175 175 175 F F F Max gen: 103, Max flush: 176 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.621505279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:30.632203096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.632248009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.642563323Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.642857201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.642895992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.642908066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.642926011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"Crucible 6e741801-bdfb-4591-ad51-616e0f05c973 has session id: edc671a0-71be-4bd4-9caa-684769b51881","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.6432932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Upstairs opts: Upstairs UUID: 6e741801-bdfb-4591-ad51-616e0f05c973, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643328051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Crucible stats registered with UUID: 6e741801-bdfb-4591-ad51-616e0f05c973","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643343545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:30.6433571Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643508665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643546576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643562461Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643697921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643715756Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"6e741801-bdfb-4591-ad51-616e0f05c973 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643746566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.64376134Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643775714Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643790139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643803753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643818487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643832931Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"ds_connection connected from Ok(127.0.0.1:41837)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.643876404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"ds_connection connected from Ok(127.0.0.1:33651)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.644022918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"ds_connection connected from Ok(127.0.0.1:42058)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.644051176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.644648764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.644708162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.644729668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645129828Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[0]R flush_numbers[0..12]: [172, 172, 167, 172, 173, 174, 168, 173, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645156576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[0]R generation[0..12]: [101, 101, 98, 101, 102, 102, 99, 102, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645173101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645190746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[1]R flush_numbers[0..12]: [172, 172, 167, 172, 173, 174, 168, 173, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645206921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[1]R generation[0..12]: [101, 101, 98, 101, 102, 102, 99, 102, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645221835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[2]R flush_numbers[0..12]: [172, 172, 167, 172, 173, 174, 168, 173, 164, 172, 168, 166]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645253565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[2]R generation[0..12]: [101, 101, 98, 101, 102, 102, 99, 102, 96, 101, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645269029Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645285114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645298708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:24:30.645312752Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645334599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645348343Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"":"downstairs","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645362627Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645375131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"0","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645389175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"1","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645403109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"client":"2","":"downstairs","session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645417303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645434468Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"6e741801-bdfb-4591-ad51-616e0f05c973 is now active with session: edc671a0-71be-4bd4-9caa-684769b51881","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645449543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645462397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667,"session_id":"edc671a0-71be-4bd4-9caa-684769b51881"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645475961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} {"msg":"Activated sub_volume 6e741801-bdfb-4591-ad51-616e0f05c973","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:30.645487224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 356 len 16384 data: 12 10 8 6 02/30 Read block 395 len 24576 03/30 Read block 335 len 40960 04/30 Write block 592 len 16384 data: 8 10 7 6 05/30 Read block 234 len 12288 06/30 Write block 172 len 12288 data: 6 7 9 07/30 Read block 555 len 4096 08/30 Write block 419 len 8192 data: 6 7 09/30 Read block 18 len 16384 10/30 Write block 289 len 24576 data: 11 11 9 9 8 11 11/30 Write block 428 len 32768 data: 8 6 6 8 8 9 8 10 12/30 Write block 262 len 12288 data: 8 7 7 13/30 Write block 548 len 36864 data: 9 8 8 8 8 9 11 10 12 14/30 Read block 537 len 24576 15/30 Write block 487 len 36864 data: 8 7 9 10 11 12 10 10 8 16/30 Read block 515 len 28672 17/30 Read block 485 len 20480 18/30 Read block 492 len 12288 19/30 Read block 357 len 4096 20/30 Read block 506 len 20480 21/30 Write block 581 len 28672 data: 10 10 10 8 8 9 7 22/30 Write block 139 len 40960 data: 7 8 7 8 8 7 6 5 6 8 23/30 Flush 24/30 Write block 226 len 28672 data: 10 8 6 5 6 8 10 25/30 Flush 26/30 Flush 27/30 Write block 423 len 36864 data: 9 9 9 8 9 9 7 7 9 28/30 Write block 54 len 32768 data: 9 9 9 10 9 8 8 8 29/30 Write block 257 len 28672 data: 11 10 9 9 8 9 8 30/30 Write block 138 len 20480 data: 10 8 9 8 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:30.657791Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1667} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 F F F 1 020-039 101 101 101 172 172 172 F F F 2 040-059 98 98 98 167 167 167 F T T <--- 3 060-079 101 101 101 172 172 172 F T T <--- 4 080-099 102 102 102 173 173 173 F F F 5 100-119 102 102 102 174 174 174 F F F 6 120-139 99 104 104 168 177 177 F T T <--- 7 140-159 102 104 104 173 177 177 F T T <--- 8 160-179 96 104 104 164 177 177 F F F <--- 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 98 104 104 166 178 178 F F F <--- 12 240-259 101 101 101 172 172 172 F T T <--- 13 260-279 99 104 104 168 177 177 F T T <--- 14 280-299 103 104 104 176 177 177 F F F <--- 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 103 104 104 176 177 177 F F F <--- 18 360-379 102 102 102 175 175 175 F F F 19 380-399 103 103 103 176 176 176 F F F 20 400-419 96 104 104 161 177 177 F F F <--- 21 420-439 100 104 104 171 177 177 F T T <--- 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 89 104 104 148 177 177 F F F <--- 25 500-519 100 100 100 170 170 170 F F F 26 520-539 103 103 103 176 176 176 F F F 27 540-559 103 104 104 176 177 177 F F F <--- 28 560-579 102 102 102 175 175 175 F F F 29 580-599 102 104 104 175 177 177 F F F <--- Max gen: 104, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 F F F 1 020-039 101 101 101 172 172 172 F F F 2 040-059 105 105 105 179 179 179 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 102 102 102 173 173 173 F F F 5 100-119 102 102 102 174 174 174 F F F 6 120-139 105 105 105 179 179 179 F F F 7 140-159 105 105 105 179 179 179 F F F 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 104 104 104 178 178 178 F F F 12 240-259 105 105 105 179 179 179 F F F 13 260-279 105 105 105 179 179 179 F F F 14 280-299 104 104 104 177 177 177 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 102 102 102 175 175 175 F F F 19 380-399 103 103 103 176 176 176 F F F 20 400-419 104 104 104 177 177 177 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 100 100 100 170 170 170 F F F 26 520-539 103 103 103 176 176 176 F F F 27 540-559 104 104 104 177 177 177 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 104 104 104 177 177 177 F F F Max gen: 105, Max flush: 179 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:44.993689301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.005503131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.005808232Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.005847004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.005863228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.005881464Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"Crucible 4c444a8f-2f47-4a5e-bbe6-43d6258a0a87 has session id: 0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006186415Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Upstairs opts: Upstairs UUID: 4c444a8f-2f47-4a5e-bbe6-43d6258a0a87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006221685Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Crucible stats registered with UUID: 4c444a8f-2f47-4a5e-bbe6-43d6258a0a87","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00623785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:45.006251744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00640429Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006428657Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006443441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006593806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00660859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"4c444a8f-2f47-4a5e-bbe6-43d6258a0a87 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006631617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006644601Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006658865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.0066731Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006689535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006704069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006718673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48552)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006822414Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62826)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006877931Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59571)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.006905589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.007617541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00764635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.007663735Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008031915Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[0]R flush_numbers[0..12]: [172, 172, 179, 179, 173, 174, 179, 179, 177, 172, 168, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008054471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[0]R generation[0..12]: [101, 101, 105, 105, 102, 102, 105, 105, 104, 101, 99, 104]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008070676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008087021Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[1]R flush_numbers[0..12]: [172, 172, 179, 179, 173, 174, 179, 179, 177, 172, 168, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008102576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[1]R generation[0..12]: [101, 101, 105, 105, 102, 102, 105, 105, 104, 101, 99, 104]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008120951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008137236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[2]R flush_numbers[0..12]: [172, 172, 179, 179, 173, 174, 179, 179, 177, 172, 168, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008152861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[2]R generation[0..12]: [101, 101, 105, 105, 102, 102, 105, 105, 104, 101, 99, 104]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008168545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00818475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008198294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008212658Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008231424Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008244918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"":"downstairs","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008259072Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008271746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"0","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008286861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"1","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008301095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"client":"2","":"downstairs","session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008315619Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:24:45.008328693Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"4c444a8f-2f47-4a5e-bbe6-43d6258a0a87 is now active with session: 0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008347088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008359982Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676,"session_id":"0b0e723f-15d3-4ea4-b0e3-38d4e2d6029a"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.008373126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} {"msg":"Activated sub_volume 4c444a8f-2f47-4a5e-bbe6-43d6258a0a87","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:45.00838452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 506 len 20480 data: 13 14 12 10 12 02/30 Write block 27 len 40960 data: 7 6 7 7 6 7 7 8 8 8 03/30 Read block 28 len 28672 04/30 Read block 343 len 40960 05/30 Write block 539 len 4096 data: 11 06/30 Write block 229 len 16384 data: 6 7 9 11 07/30 Read block 419 len 40960 08/30 Read block 348 len 20480 09/30 Read block 439 len 12288 10/30 Read block 516 len 4096 11/30 Write block 113 len 32768 data: 14 13 11 7 8 6 6 5 12/30 Read block 519 len 40960 13/30 Read block 309 len 20480 14/30 Read block 110 len 20480 15/30 Write block 376 len 40960 data: 8 5 7 7 9 12 9 6 5 11 16/30 Read block 586 len 32768 17/30 Read block 338 len 4096 18/30 Write block 387 len 20480 data: 10 8 8 8 8 19/30 Read block 497 len 16384 20/30 Read block 419 len 4096 21/30 Read block 379 len 16384 22/30 Read block 220 len 28672 23/30 Read block 15 len 28672 24/30 Read block 42 len 24576 25/30 Read block 194 len 12288 26/30 Write block 140 len 12288 data: 10 9 10 27/30 Read block 143 len 16384 28/30 Write block 505 len 20480 data: 14 14 15 13 11 29/30 Write block 4 len 16384 data: 3 4 5 5 30/30 Write block 44 len 40960 data: 9 9 9 12 13 12 11 8 9 9 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:45.020575412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1676} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 172 172 172 T F T <--- 1 020-039 101 101 101 172 172 172 T T T 2 040-059 105 105 105 179 179 179 T F T <--- 3 060-079 105 105 105 179 179 179 F F F 4 080-099 102 102 102 173 173 173 F F F 5 100-119 102 102 102 174 174 174 T F T <--- 6 120-139 105 105 105 179 179 179 T F T <--- 7 140-159 105 105 105 179 179 179 T F T <--- 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 104 104 104 178 178 178 T F T <--- 12 240-259 105 105 105 179 179 179 F F F 13 260-279 105 105 105 179 179 179 F F F 14 280-299 104 104 104 177 177 177 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 102 102 102 175 175 175 T F T <--- 19 380-399 103 103 103 176 176 176 T F T <--- 20 400-419 104 104 104 177 177 177 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 100 100 100 170 170 170 T T T 26 520-539 103 103 103 176 176 176 T F T <--- 27 540-559 104 104 104 177 177 177 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 104 104 104 177 177 177 F F F Max gen: 105, Max flush: 179 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 180 180 180 F F F 1 020-039 106 106 106 180 180 180 F F F 2 040-059 106 106 106 180 180 180 F F F 3 060-079 105 105 105 179 179 179 F F F 4 080-099 102 102 102 173 173 173 F F F 5 100-119 106 106 106 180 180 180 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 106 106 106 180 180 180 F F F 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 F F F 12 240-259 105 105 105 179 179 179 F F F 13 260-279 105 105 105 179 179 179 F F F 14 280-299 104 104 104 177 177 177 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 106 106 106 180 180 180 F F F 19 380-399 106 106 106 180 180 180 F F F 20 400-419 104 104 104 177 177 177 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 106 106 106 180 180 180 F F F 27 540-559 104 104 104 177 177 177 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 104 104 104 177 177 177 F F F Max gen: 106, Max flush: 180 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.347901151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.359625285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.359931366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.359968577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.359980661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.359998396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"Crucible b7523bd1-7417-40ce-9c10-5ddc037186ad has session id: 61306881-6e75-4063-8996-51649da203ce","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360331105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Upstairs opts: Upstairs UUID: b7523bd1-7417-40ce-9c10-5ddc037186ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360366346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Crucible stats registered with UUID: b7523bd1-7417-40ce-9c10-5ddc037186ad","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.36038237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:59.360397665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360564295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360589232Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360603966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360767635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360787661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"b7523bd1-7417-40ce-9c10-5ddc037186ad active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360800655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360813829Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360828243Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360843288Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360857632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360872486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360886861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:45600)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360932814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:64074)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360949609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:41946)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.360964704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.36162297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.361711466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.361950968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362302142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 180, 179, 173, 180, 180, 180, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.36232764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[0]R generation[0..12]: [106, 106, 106, 105, 102, 106, 106, 106, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362343655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.3623601Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 180, 179, 173, 180, 180, 180, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362376355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[1]R generation[0..12]: [106, 106, 106, 105, 102, 106, 106, 106, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362391759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362407834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 180, 179, 173, 180, 180, 180, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362423699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[2]R generation[0..12]: [106, 106, 106, 105, 102, 106, 106, 106, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362439193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362455578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Max found gen is 107","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362468992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Generation requested: 108 >= found:107","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:24:59.362483216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362504833Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362518097Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"":"downstairs","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362532441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362545295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"0","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362559579Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"1","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362573843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"client":"2","":"downstairs","session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362588258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362603902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"b7523bd1-7417-40ce-9c10-5ddc037186ad is now active with session: 61306881-6e75-4063-8996-51649da203ce","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362618837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362631371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685,"session_id":"61306881-6e75-4063-8996-51649da203ce"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362644744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} {"msg":"Activated sub_volume b7523bd1-7417-40ce-9c10-5ddc037186ad","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:24:59.362655938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 373 len 36864 data: 10 9 8 9 6 8 8 10 13 03/30 Write block 95 len 32768 data: 7 7 9 8 8 11 10 13 04/30 Read block 145 len 4096 05/30 Read block 438 len 8192 06/30 Flush 07/30 Write block 146 len 32768 data: 6 7 9 9 9 7 8 9 08/30 Read block 380 len 4096 09/30 Write block 590 len 40960 data: 9 9 9 11 8 7 6 5 5 4 10/30 Flush 11/30 Write block 551 len 36864 data: 9 9 10 12 11 13 11 12 10 12/30 Read block 230 len 36864 13/30 Write block 32 len 4096 data: 8 14/30 Read block 102 len 36864 15/30 Read block 210 len 32768 16/30 Read block 400 len 4096 17/30 Write block 245 len 20480 data: 6 7 7 10 12 18/30 Read block 457 len 28672 19/30 Write block 241 len 32768 data: 15 11 9 8 7 8 8 11 20/30 Read block 99 len 4096 21/30 Read block 540 len 12288 22/30 Write block 72 len 20480 data: 7 8 8 8 10 23/30 Read block 576 len 36864 24/30 Read block 281 len 20480 25/30 Read block 425 len 36864 26/30 Write block 525 len 32768 data: 5 5 9 9 9 9 11 11 27/30 Read block 262 len 32768 28/30 Read block 477 len 8192 29/30 Read block 8 len 32768 30/30 Write block 84 len 32768 data: 7 6 7 6 6 9 7 6 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1000)) Some(JobId(1009)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:24:59.376032263Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1685} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 180 180 180 F F F 1 020-039 106 106 106 180 180 180 T F T <--- 2 040-059 106 106 106 180 180 180 F F F 3 060-079 105 105 105 179 179 179 T F T <--- 4 080-099 108 102 108 182 173 182 T F T <--- 5 100-119 108 106 108 182 180 182 F F F <--- 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 106 108 183 180 183 F F F <--- 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 F F F 12 240-259 105 105 105 179 179 179 T F T <--- 13 260-279 105 105 105 179 179 179 F F F 14 280-299 104 104 104 177 177 177 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 106 108 182 180 182 F T F <--- 19 380-399 108 106 108 182 180 182 F T F <--- 20 400-419 104 104 104 177 177 177 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 106 106 106 180 180 180 T F T <--- 27 540-559 104 104 104 177 177 177 T F T <--- 28 560-579 102 102 102 175 175 175 F F F 29 580-599 108 104 108 183 177 183 F F F <--- Max gen: 108, Max flush: 183 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 180 180 180 F F F 1 020-039 109 109 109 184 184 184 F F F 2 040-059 106 106 106 180 180 180 F F F 3 060-079 109 109 109 184 184 184 F F F 4 080-099 109 109 109 184 184 184 F F F 5 100-119 108 108 108 182 182 182 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 F F F 12 240-259 109 109 109 184 184 184 F F F 13 260-279 105 105 105 179 179 179 F F F 14 280-299 104 104 104 177 177 177 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 108 108 108 182 182 182 F F F 20 400-419 104 104 104 177 177 177 F F F 21 420-439 105 105 105 179 179 179 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 109 109 109 184 184 184 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 108 108 108 183 183 183 F F F Max gen: 109, Max flush: 184 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.717726154Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.729634112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.729947795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.729985896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730002101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730020277Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"Crucible 2a41970a-5ee1-4579-a1fb-e1f16b2efca9 has session id: 463127b9-3b90-481d-ad8a-6f59c71540f7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730375783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Upstairs opts: Upstairs UUID: 2a41970a-5ee1-4579-a1fb-e1f16b2efca9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730411843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Crucible stats registered with UUID: 2a41970a-5ee1-4579-a1fb-e1f16b2efca9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730428178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:13.730441612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730618815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730642642Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730657296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730797918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730816304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"2a41970a-5ee1-4579-a1fb-e1f16b2efca9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730839011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730852095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730866099Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730880953Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730897478Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730957256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730980163Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42224)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.730995647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37775)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.731010602Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34693)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.731125556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.73171042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.731779581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.731835998Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.73221277Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[0]R flush_numbers[0..12]: [180, 184, 180, 184, 184, 182, 180, 183, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732235797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[0]R generation[0..12]: [106, 109, 106, 109, 109, 108, 106, 108, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732251822Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732268487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[1]R flush_numbers[0..12]: [180, 184, 180, 184, 184, 182, 180, 183, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732284551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[1]R generation[0..12]: [106, 109, 106, 109, 109, 108, 106, 108, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732302177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732318902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[2]R flush_numbers[0..12]: [180, 184, 180, 184, 184, 182, 180, 183, 177, 172, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732334726Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[2]R generation[0..12]: [106, 109, 106, 109, 109, 108, 106, 108, 104, 101, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732350491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732366996Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.73238051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:25:13.732394664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732411359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732424993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"":"downstairs","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732439488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732452381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"0","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732466486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"1","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.73248062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"client":"2","":"downstairs","session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732495104Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732511329Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"2a41970a-5ee1-4579-a1fb-e1f16b2efca9 is now active with session: 463127b9-3b90-481d-ad8a-6f59c71540f7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732528264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732541238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694,"session_id":"463127b9-3b90-481d-ad8a-6f59c71540f7"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732554632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} {"msg":"Activated sub_volume 2a41970a-5ee1-4579-a1fb-e1f16b2efca9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:13.732566215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 410 len 20480 data: 6 6 7 6 9 02/30 Write block 287 len 24576 data: 10 11 12 12 10 10 03/30 Write block 391 len 4096 data: 9 04/30 Read block 216 len 28672 05/30 Read block 576 len 8192 06/30 Write block 69 len 28672 data: 11 10 8 8 9 9 9 07/30 Read block 18 len 24576 08/30 Read block 267 len 16384 09/30 Read block 86 len 8192 10/30 Flush 11/30 Flush 12/30 Flush 13/30 Write block 425 len 8192 data: 10 9 14/30 Write block 257 len 32768 data: 12 11 10 10 9 10 9 8 15/30 Read block 176 len 28672 16/30 Read block 237 len 28672 17/30 Read block 300 len 32768 18/30 Read block 318 len 4096 19/30 Read block 474 len 40960 20/30 Write block 81 len 40960 data: 8 8 9 8 7 8 7 7 10 8 21/30 Write block 187 len 4096 data: 8 22/30 Read block 329 len 4096 23/30 Write block 332 len 12288 data: 4 5 7 24/30 Write block 275 len 40960 data: 10 8 9 9 9 11 8 7 8 9 25/30 Read block 181 len 4096 26/30 Read block 476 len 32768 27/30 Read block 135 len 40960 28/30 Read block 172 len 28672 29/30 Read block 431 len 36864 30/30 Read block 444 len 28672 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 6 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:13.744373643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1694} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 180 180 180 F F F 1 020-039 109 109 109 184 184 184 F F F 2 040-059 106 106 106 180 180 180 F F F 3 060-079 110 110 109 185 185 184 F F F <--- 4 080-099 109 109 109 184 184 184 T T F <--- 5 100-119 108 108 108 182 182 182 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 104 104 104 177 177 177 F F F 9 180-199 101 101 101 172 172 172 T T F <--- 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 F F F 12 240-259 109 109 109 184 184 184 T T F <--- 13 260-279 105 105 105 179 179 179 T T F <--- 14 280-299 110 110 104 185 185 177 T T F <--- 15 300-319 102 102 102 175 175 175 F F F 16 320-339 103 103 103 176 176 176 T T F <--- 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 110 110 108 185 185 182 F F F <--- 20 400-419 110 110 104 185 185 177 F F T <--- 21 420-439 105 105 105 179 179 179 T T F <--- 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 109 109 109 184 184 184 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 108 108 108 183 183 183 F F F Max gen: 110, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 180 180 180 F F F 1 020-039 109 109 109 184 184 184 F F F 2 040-059 106 106 106 180 180 180 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 108 108 108 182 182 182 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 104 104 104 177 177 177 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 F F F 12 240-259 111 111 111 186 186 186 F F F 13 260-279 111 111 111 186 186 186 F F F 14 280-299 111 111 111 186 186 186 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 111 111 111 186 186 186 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 110 110 110 185 185 185 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 111 111 111 186 186 186 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 106 106 106 180 180 180 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 109 109 109 184 184 184 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 108 108 108 183 183 183 F F F Max gen: 111, Max flush: 186 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.06892059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.080686946Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.080998949Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081061608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081075092Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081092957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"Crucible b7843f11-705a-4e02-b76c-6b80a3e1a5d1 has session id: 29166118-48c7-41fe-95a2-7b0d0ccfee44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081424856Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Upstairs opts: Upstairs UUID: b7843f11-705a-4e02-b76c-6b80a3e1a5d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081459987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Crucible stats registered with UUID: b7843f11-705a-4e02-b76c-6b80a3e1a5d1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081475541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:28.081489355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081654335Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081681813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081696967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081850013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081866638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"b7843f11-705a-4e02-b76c-6b80a3e1a5d1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081890265Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081903169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081917223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081931307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081944781Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.081958865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.08197297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"ds_connection connected from Ok(127.0.0.1:57265)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082070589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"ds_connection connected from Ok(127.0.0.1:47905)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082098567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"ds_connection connected from Ok(127.0.0.1:37281)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082197106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082805758Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082834946Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.082862535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083279459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[0]R flush_numbers[0..12]: [180, 184, 180, 185, 186, 182, 180, 183, 177, 186, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083301655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[0]R generation[0..12]: [106, 109, 106, 110, 111, 108, 106, 108, 104, 111, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.08331732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083334135Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[1]R flush_numbers[0..12]: [180, 184, 180, 185, 186, 182, 180, 183, 177, 186, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.0833502Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[1]R generation[0..12]: [106, 109, 106, 110, 111, 108, 106, 108, 104, 111, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083366005Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.08338266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[2]R flush_numbers[0..12]: [180, 184, 180, 185, 186, 182, 180, 183, 177, 186, 168, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083398814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[2]R generation[0..12]: [106, 109, 106, 110, 111, 108, 106, 108, 104, 111, 99, 106]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083414549Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083430874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083444338Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083458272Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083471776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:25:28.083487251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"":"downstairs","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083506857Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.08351971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"0","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083533895Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"1","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083547689Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"client":"2","":"downstairs","session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083561963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083575297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"b7843f11-705a-4e02-b76c-6b80a3e1a5d1 is now active with session: 29166118-48c7-41fe-95a2-7b0d0ccfee44","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083590341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:25:28.083603145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703,"session_id":"29166118-48c7-41fe-95a2-7b0d0ccfee44"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.08361802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} {"msg":"Activated sub_volume b7843f11-705a-4e02-b76c-6b80a3e1a5d1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:28.083630253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 8192 data: 9 8 02/30 Read block 18 len 32768 03/30 Write block 5 len 36864 data: 5 6 6 6 8 8 10 10 10 04/30 Read block 140 len 36864 05/30 Flush 06/30 Flush 07/30 Write block 394 len 36864 data: 9 6 7 7 8 8 9 11 12 08/30 Read block 75 len 8192 09/30 Write block 281 len 32768 data: 9 8 9 10 10 10 11 12 10/30 Read block 380 len 4096 11/30 Write block 9 len 36864 data: 9 9 11 11 11 9 10 10 10 12/30 Write block 244 len 36864 data: 9 8 9 9 12 13 13 9 8 13/30 Write block 504 len 40960 data: 12 15 15 16 14 12 13 12 11 12 14/30 Write block 586 len 24576 data: 10 8 7 5 10 10 15/30 Read block 543 len 20480 16/30 Write block 226 len 40960 data: 11 9 7 7 8 10 12 12 13 15 17/30 Read block 520 len 12288 18/30 Write block 308 len 24576 data: 9 8 8 6 5 6 19/30 Write block 270 len 12288 data: 12 13 17 20/30 Write block 162 len 32768 data: 4 6 6 6 5 5 5 3 21/30 Write block 517 len 8192 data: 12 10 22/30 Read block 85 len 4096 23/30 Read block 11 len 16384 24/30 Read block 26 len 28672 25/30 Read block 402 len 36864 26/30 Write block 19 len 16384 data: 9 6 7 6 27/30 Read block 245 len 16384 28/30 Write block 504 len 24576 data: 13 16 16 17 15 13 29/30 Read block 590 len 20480 30/30 Read block 420 len 40960 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) None Some(JobId(1005)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:28.096963546Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1703} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 106 112 187 180 187 T F T <--- 1 020-039 109 109 109 184 184 184 T F T <--- 2 040-059 106 106 106 180 180 180 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 108 108 108 182 182 182 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 104 104 104 177 177 177 T F T <--- 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 106 106 106 180 180 180 T F T <--- 12 240-259 111 111 111 186 186 186 T F T <--- 13 260-279 111 111 111 186 186 186 T F T <--- 14 280-299 111 111 111 186 186 186 T F T <--- 15 300-319 102 102 102 175 175 175 T F T <--- 16 320-339 111 111 111 186 186 186 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 110 110 110 185 185 185 T F T <--- 20 400-419 110 110 110 185 185 185 T F T <--- 21 420-439 111 111 111 186 186 186 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 112 106 112 187 180 187 T F T <--- 26 520-539 109 109 109 184 184 184 F F F 27 540-559 109 109 109 184 184 184 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 108 108 108 183 183 183 T F T <--- Max gen: 112, Max flush: 187 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 188 188 188 F F F 1 020-039 113 113 113 188 188 188 F F F 2 040-059 106 106 106 180 180 180 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 108 108 108 182 182 182 F F F 6 120-139 106 106 106 180 180 180 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 113 113 113 188 188 188 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 113 113 113 188 188 188 F F F 12 240-259 113 113 113 188 188 188 F F F 13 260-279 113 113 113 188 188 188 F F F 14 280-299 113 113 113 188 188 188 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 111 111 111 186 186 186 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 113 113 113 188 188 188 F F F 20 400-419 113 113 113 188 188 188 F F F 21 420-439 111 111 111 186 186 186 F F F 22 440-459 99 99 99 168 168 168 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 104 104 104 177 177 177 F F F 25 500-519 113 113 113 188 188 188 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 109 109 109 184 184 184 F F F 28 560-579 102 102 102 175 175 175 F F F 29 580-599 113 113 113 188 188 188 F F F Max gen: 113, Max flush: 188 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.425356352Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437010815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437319827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437356708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437372202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437390298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"Crucible db674f63-9565-4e37-9196-e606bf99f565 has session id: 85fc372a-f672-483a-8f45-794579651c6b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437726438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Upstairs opts: Upstairs UUID: db674f63-9565-4e37-9196-e606bf99f565, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437784925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Crucible stats registered with UUID: db674f63-9565-4e37-9196-e606bf99f565","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.43780144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:42.437814904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.437985105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438020806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.43803658Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438181974Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438197778Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"db674f63-9565-4e37-9196-e606bf99f565 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438214303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438227237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438241551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438255386Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438308231Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438324166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.43833816Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36113)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438353215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50213)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438368359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52354)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.438384554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439139849Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439169208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439346601Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439669727Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[0]R flush_numbers[0..12]: [188, 188, 180, 185, 186, 182, 180, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439693294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[0]R generation[0..12]: [113, 113, 106, 110, 111, 108, 106, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439709299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439726004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[1]R flush_numbers[0..12]: [188, 188, 180, 185, 186, 182, 180, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439741788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[1]R generation[0..12]: [113, 113, 106, 110, 111, 108, 106, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439760454Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439776909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[2]R flush_numbers[0..12]: [188, 188, 180, 185, 186, 182, 180, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439793054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[2]R generation[0..12]: [113, 113, 106, 110, 111, 108, 106, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439808969Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439825553Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439839067Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-03-17T14:25:42.439859654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439873098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439886652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"":"downstairs","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439900986Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.43991393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"0","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439928184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"1","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439942218Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"client":"2","":"downstairs","session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439956773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:25:42.439970307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"db674f63-9565-4e37-9196-e606bf99f565 is now active with session: 85fc372a-f672-483a-8f45-794579651c6b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.439989662Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.440002566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712,"session_id":"85fc372a-f672-483a-8f45-794579651c6b"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.44001585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} {"msg":"Activated sub_volume db674f63-9565-4e37-9196-e606bf99f565","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:42.440027374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 12288 data: 10 13 14 02/30 Write block 288 len 28672 data: 13 13 13 11 11 9 12 03/30 Read block 469 len 32768 04/30 Read block 6 len 24576 05/30 Read block 590 len 8192 06/30 Read block 585 len 12288 07/30 Read block 404 len 24576 08/30 Write block 563 len 4096 data: 12 09/30 Read block 561 len 16384 10/30 Write block 118 len 20480 data: 7 7 6 7 8 11/30 Read block 374 len 20480 12/30 Write block 40 len 36864 data: 6 8 10 9 10 10 11 14 15 13/30 Write block 559 len 8192 data: 11 12 14/30 Write block 122 len 32768 data: 9 6 7 6 7 7 8 7 15/30 Read block 582 len 8192 16/30 Read block 444 len 12288 17/30 Write block 442 len 4096 data: 5 18/30 Read block 250 len 40960 19/30 Write block 411 len 28672 data: 7 8 7 10 7 8 6 20/30 Read block 411 len 28672 21/30 Read block 588 len 28672 22/30 Read block 370 len 28672 23/30 Flush 24/30 Write block 479 len 12288 data: 8 8 8 25/30 Read block 563 len 36864 26/30 Read block 508 len 40960 27/30 Write block 320 len 28672 data: 10 10 9 8 8 9 8 28/30 Read block 8 len 12288 29/30 Read block 245 len 36864 30/30 Write block 285 len 16384 data: 11 11 12 14 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 17 19 Done 29 29 13 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:42.452081145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 188 188 188 F F F 1 020-039 113 113 113 188 188 188 F F F 2 040-059 114 114 106 189 189 180 F F T <--- 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 114 114 108 189 189 182 F F T <--- 6 120-139 114 114 106 189 189 180 F F T <--- 7 140-159 108 108 108 183 183 183 F F F 8 160-179 113 113 113 188 188 188 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 113 113 113 188 188 188 F F F 12 240-259 113 113 113 188 188 188 F F F 13 260-279 113 113 113 188 188 188 F F F 14 280-299 114 114 113 189 189 188 T T T <--- 15 300-319 113 113 113 188 188 188 F F F 16 320-339 111 111 111 186 186 186 T T F <--- 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 113 113 113 188 188 188 F F F 20 400-419 114 114 113 189 189 188 F F F <--- 21 420-439 111 111 111 186 186 186 F F F 22 440-459 114 114 99 189 189 168 F F F <--- 23 460-479 82 82 82 134 134 134 T T F <--- 24 480-499 104 104 104 177 177 177 T T F <--- 25 500-519 113 113 113 188 188 188 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 114 114 109 189 189 184 F F T <--- 28 560-579 114 114 102 189 189 175 F F T <--- 29 580-599 113 113 113 188 188 188 F F F Max gen: 114, Max flush: 189 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 188 188 188 F F F 1 020-039 113 113 113 188 188 188 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 114 114 114 189 189 189 F F F 6 120-139 114 114 114 189 189 189 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 113 113 113 188 188 188 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 113 113 113 188 188 188 F F F 12 240-259 113 113 113 188 188 188 F F F 13 260-279 113 113 113 188 188 188 F F F 14 280-299 115 115 115 190 190 190 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 108 108 108 182 182 182 F F F 19 380-399 113 113 113 188 188 188 F F F 20 400-419 114 114 114 189 189 189 F F F 21 420-439 111 111 111 186 186 186 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 115 115 115 190 190 190 F F F 24 480-499 115 115 115 190 190 190 F F F 25 500-519 113 113 113 188 188 188 F F F 26 520-539 109 109 109 184 184 184 F F F 27 540-559 114 114 114 189 189 189 F F F 28 560-579 114 114 114 189 189 189 F F F 29 580-599 113 113 113 188 188 188 F F F Max gen: 115, Max flush: 190 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.776086552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.787733572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788076504Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788114746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788127059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788145095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"Crucible cff05f24-cd61-4773-a02f-2e9aac01b2ba has session id: 21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.78849791Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Upstairs opts: Upstairs UUID: cff05f24-cd61-4773-a02f-2e9aac01b2ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.78853355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Crucible stats registered with UUID: cff05f24-cd61-4773-a02f-2e9aac01b2ba","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788549635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:56.788563339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788715925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788744123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788759208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788893898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788910313Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"cff05f24-cd61-4773-a02f-2e9aac01b2ba active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788926687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788939391Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788953646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.78896748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788981444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.788995988Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.789010532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:62005)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.789079563Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:54794)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.789183334Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:50035)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.78920323Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.789845081Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.789895276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790015702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790358694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[0]R flush_numbers[0..12]: [188, 188, 189, 185, 186, 189, 189, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790382831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[0]R generation[0..12]: [113, 113, 114, 110, 111, 114, 114, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790398906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790415911Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[1]R flush_numbers[0..12]: [188, 188, 189, 185, 186, 189, 189, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790432126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[1]R generation[0..12]: [113, 113, 114, 110, 111, 114, 114, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790447741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790464196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[2]R flush_numbers[0..12]: [188, 188, 189, 185, 186, 189, 189, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.79047985Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[2]R generation[0..12]: [113, 113, 114, 110, 111, 114, 114, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790495435Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.79051189Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790525474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:25:56.790539708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790561325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790575009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"":"downstairs","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790589093Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790601426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"0","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790615271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"1","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790629075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"client":"2","":"downstairs","session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790643889Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790660204Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"cff05f24-cd61-4773-a02f-2e9aac01b2ba is now active with session: 21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790675399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790688302Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721,"session_id":"21dc4bc4-3fbb-4ba8-ac82-6678a64d2fee"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.790701716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} {"msg":"Activated sub_volume cff05f24-cd61-4773-a02f-2e9aac01b2ba","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:25:56.79071331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 365 len 4096 data: 7 03/30 Read block 309 len 16384 04/30 Write block 572 len 40960 data: 4 4 4 4 4 6 6 6 11 11 05/30 Write block 15 len 12288 data: 11 11 11 06/30 Read block 251 len 24576 07/30 Read block 296 len 4096 08/30 Flush 09/30 Write block 92 len 12288 data: 8 9 7 10/30 Write block 128 len 40960 data: 9 8 5 5 8 8 8 11 10 12 11/30 Read block 322 len 40960 12/30 Read block 468 len 16384 13/30 Read block 430 len 4096 14/30 Read block 257 len 28672 15/30 Write block 486 len 12288 data: 6 9 8 16/30 Write block 532 len 40960 data: 12 14 12 10 10 10 11 12 9 11 17/30 Write block 427 len 32768 data: 10 10 8 8 10 9 10 9 18/30 Read block 46 len 28672 19/30 Read block 173 len 40960 20/30 Write block 550 len 40960 data: 9 10 10 11 13 12 14 12 13 12 21/30 Write block 422 len 16384 data: 8 10 10 11 22/30 Read block 5 len 4096 23/30 Write block 259 len 36864 data: 11 11 10 11 10 9 7 10 14 24/30 Read block 456 len 4096 25/30 Write block 391 len 32768 data: 10 9 7 10 7 8 8 9 26/30 Write block 28 len 36864 data: 7 8 8 7 9 8 9 9 9 27/30 Read block 360 len 32768 28/30 Read block 148 len 20480 29/30 Read block 543 len 32768 30/30 Write block 397 len 24576 data: 9 10 9 10 12 13 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 23 25 Done 28 28 6 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:25:56.803459888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1721} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 113 192 192 188 F F T <--- 1 020-039 113 113 113 188 188 188 T T F <--- 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 111 111 111 186 186 186 T T F <--- 5 100-119 114 114 114 189 189 189 F F F 6 120-139 114 114 114 189 189 189 T T F <--- 7 140-159 108 108 108 183 183 183 F F F 8 160-179 113 113 113 188 188 188 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 113 113 113 188 188 188 F F F 12 240-259 113 113 113 188 188 188 T T F <--- 13 260-279 113 113 113 188 188 188 T T F <--- 14 280-299 115 115 115 190 190 190 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 116 116 108 192 192 182 F F T <--- 19 380-399 113 113 113 188 188 188 T T F <--- 20 400-419 114 114 114 189 189 189 T T F <--- 21 420-439 111 111 111 186 186 186 T T F <--- 22 440-459 114 114 114 189 189 189 F F F 23 460-479 115 115 115 190 190 190 F F F 24 480-499 115 115 115 190 190 190 T T F <--- 25 500-519 113 113 113 188 188 188 F F F 26 520-539 109 109 109 184 184 184 T T F <--- 27 540-559 114 114 114 189 189 189 T T F <--- 28 560-579 116 116 114 192 192 189 F F T <--- 29 580-599 116 116 113 192 192 188 F F T <--- Max gen: 116, Max flush: 192 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 192 192 192 F F F 1 020-039 117 117 117 193 193 193 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 117 117 117 193 193 193 F F F 5 100-119 114 114 114 189 189 189 F F F 6 120-139 117 117 117 193 193 193 F F F 7 140-159 108 108 108 183 183 183 F F F 8 160-179 113 113 113 188 188 188 F F F 9 180-199 111 111 111 186 186 186 F F F 10 200-219 99 99 99 168 168 168 F F F 11 220-239 113 113 113 188 188 188 F F F 12 240-259 117 117 117 193 193 193 F F F 13 260-279 117 117 117 193 193 193 F F F 14 280-299 115 115 115 190 190 190 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 104 104 104 177 177 177 F F F 18 360-379 116 116 116 192 192 192 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 117 117 117 193 193 193 F F F 21 420-439 117 117 117 193 193 193 F F F 22 440-459 114 114 114 189 189 189 F F F 23 460-479 115 115 115 190 190 190 F F F 24 480-499 117 117 117 193 193 193 F F F 25 500-519 113 113 113 188 188 188 F F F 26 520-539 117 117 117 193 193 193 F F F 27 540-559 117 117 117 193 193 193 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 117, Max flush: 193 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.14345687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155156506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155487305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155525596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155541361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155559426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"Crucible 2791e81f-0c64-4cc9-b9c8-ff373c067a5c has session id: ede25592-e74c-45fe-adc5-93443164463c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155915522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Upstairs opts: Upstairs UUID: 2791e81f-0c64-4cc9-b9c8-ff373c067a5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155950783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Crucible stats registered with UUID: 2791e81f-0c64-4cc9-b9c8-ff373c067a5c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.155966418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:11.155979742Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156140219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156164607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156179641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.15631095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156326405Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"2791e81f-0c64-4cc9-b9c8-ff373c067a5c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156350032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156363226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.15637761Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156391474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156407979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156422804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156437268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40309)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156482901Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33432)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156499566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38329)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.156514661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.157296044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.157374147Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.157716659Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158078607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[0]R flush_numbers[0..12]: [192, 193, 189, 185, 193, 189, 193, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158102384Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[0]R generation[0..12]: [116, 117, 114, 110, 117, 114, 117, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158118969Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158136024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[1]R flush_numbers[0..12]: [192, 193, 189, 185, 193, 189, 193, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158151779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[1]R generation[0..12]: [116, 117, 114, 110, 117, 114, 117, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158170474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158187039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[2]R flush_numbers[0..12]: [192, 193, 189, 185, 193, 189, 193, 183, 188, 186, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158202604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[2]R generation[0..12]: [116, 117, 114, 110, 117, 114, 117, 108, 113, 111, 99, 113]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158218599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158235804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158249408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Generation requested: 118 >= found:118","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2026-03-17T14:26:11.158270554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158284308Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158298612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"":"downstairs","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158313577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158326321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"0","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158340785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"1","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158354739Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"client":"2","":"downstairs","session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158368883Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158385408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"2791e81f-0c64-4cc9-b9c8-ff373c067a5c is now active with session: ede25592-e74c-45fe-adc5-93443164463c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158402663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158415597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730,"session_id":"ede25592-e74c-45fe-adc5-93443164463c"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158429111Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} {"msg":"Activated sub_volume 2791e81f-0c64-4cc9-b9c8-ff373c067a5c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:11.158440525Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 20480 data: 8 8 9 11 13 02/30 Read block 373 len 8192 03/30 Write block 219 len 12288 data: 9 9 11 04/30 Write block 147 len 4096 data: 8 05/30 Read block 313 len 20480 06/30 Read block 79 len 36864 07/30 Write block 451 len 40960 data: 9 9 8 4 6 5 5 4 4 5 08/30 Read block 517 len 4096 09/30 Read block 4 len 28672 10/30 Read block 443 len 36864 11/30 Write block 295 len 16384 data: 10 11 9 10 12/30 Write block 236 len 4096 data: 13 13/30 Write block 139 len 4096 data: 9 14/30 Write block 504 len 36864 data: 14 17 17 18 16 14 14 13 12 15/30 Write block 226 len 20480 data: 12 10 9 9 10 16/30 Write block 453 len 8192 data: 9 5 17/30 Read block 29 len 4096 18/30 Write block 451 len 4096 data: 10 19/30 Read block 225 len 24576 20/30 Write block 8 len 4096 data: 7 21/30 Read block 395 len 8192 22/30 Read block 227 len 20480 23/30 Read block 157 len 32768 24/30 Write block 344 len 8192 data: 14 12 25/30 Write block 456 len 28672 data: 6 6 5 5 6 7 9 26/30 Write block 176 len 40960 data: 13 10 8 10 9 6 8 7 6 7 27/30 Write block 353 len 32768 data: 15 14 15 13 11 9 7 9 28/30 Write block 476 len 16384 data: 8 8 8 9 29/30 Read block 426 len 12288 30/30 Read block 166 len 20480 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:11.168981245Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1730} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 192 192 192 T F T <--- 1 020-039 117 117 117 193 193 193 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 117 117 117 193 193 193 F F F 5 100-119 114 114 114 189 189 189 F F F 6 120-139 117 117 117 193 193 193 T F T <--- 7 140-159 108 108 108 183 183 183 T F T <--- 8 160-179 113 113 113 188 188 188 T F T <--- 9 180-199 111 111 111 186 186 186 T F T <--- 10 200-219 99 99 99 168 168 168 T F T <--- 11 220-239 113 113 113 188 188 188 T T T 12 240-259 117 117 117 193 193 193 F F F 13 260-279 117 117 117 193 193 193 F F F 14 280-299 115 115 115 190 190 190 T F T <--- 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 104 104 104 177 177 177 T F T <--- 18 360-379 116 116 116 192 192 192 T F T <--- 19 380-399 117 117 117 193 193 193 F F F 20 400-419 117 117 117 193 193 193 F F F 21 420-439 117 117 117 193 193 193 F F F 22 440-459 114 114 114 189 189 189 T F T <--- 23 460-479 115 115 115 190 190 190 T F T <--- 24 480-499 117 117 117 193 193 193 F F F 25 500-519 113 113 113 188 188 188 T F T <--- 26 520-539 117 117 117 193 193 193 F F F 27 540-559 117 117 117 193 193 193 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 117, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 F F F 1 020-039 117 117 117 193 193 193 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 117 117 117 193 193 193 F F F 5 100-119 114 114 114 189 189 189 F F F 6 120-139 118 118 118 194 194 194 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 118 118 118 194 194 194 F F F 9 180-199 118 118 118 194 194 194 F F F 10 200-219 118 118 118 194 194 194 F F F 11 220-239 118 118 118 194 194 194 F F F 12 240-259 117 117 117 193 193 193 F F F 13 260-279 117 117 117 193 193 193 F F F 14 280-299 118 118 118 194 194 194 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 118 118 118 194 194 194 F F F 18 360-379 118 118 118 194 194 194 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 117 117 117 193 193 193 F F F 21 420-439 117 117 117 193 193 193 F F F 22 440-459 118 118 118 194 194 194 F F F 23 460-479 118 118 118 194 194 194 F F F 24 480-499 117 117 117 193 193 193 F F F 25 500-519 118 118 118 194 194 194 F F F 26 520-539 117 117 117 193 193 193 F F F 27 540-559 117 117 117 193 193 193 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 118, Max flush: 194 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.503763975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.515581896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.515885237Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.515922428Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.515934411Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.515952156Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"Crucible a514ce27-8f9b-485e-8f62-f705b24836dd has session id: 2f24efc5-a138-4b2b-9b68-7088da4dfb51","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516310673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Upstairs opts: Upstairs UUID: a514ce27-8f9b-485e-8f62-f705b24836dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516335951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Crucible stats registered with UUID: a514ce27-8f9b-485e-8f62-f705b24836dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516351055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:25.516364509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516548564Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516596498Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516612183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516742552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516762028Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"a514ce27-8f9b-485e-8f62-f705b24836dd active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516779253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516792087Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516806521Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516820955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516834569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516849244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516863668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:51592)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.516955235Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:54067)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.517029077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:56785)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.517045342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.51774392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.517773629Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.517790664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518139748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[0]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 189, 194, 194, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518161665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[0]R generation[0..12]: [118, 117, 114, 110, 117, 114, 118, 118, 118, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.51817812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518194935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[1]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 189, 194, 194, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518210769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[1]R generation[0..12]: [118, 117, 114, 110, 117, 114, 118, 118, 118, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518226504Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518242899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[2]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 189, 194, 194, 194, 194, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518258604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[2]R generation[0..12]: [118, 117, 114, 110, 117, 114, 118, 118, 118, 118, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518275199Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518291864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518305528Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Generation requested: 120 >= found:119","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518325864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518341628Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518355763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"":"downstairs","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518370137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518382831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"0","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518396665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"1","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518410449Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"client":"2","":"downstairs","session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518424423Z",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518441118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"a514ce27-8f9b-485e-8f62-f705b24836dd is now active with session: 2f24efc5-a138-4b2b-9b68-7088da4dfb51","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518455993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518468456Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739,"session_id":"2f24efc5-a138-4b2b-9b68-7088da4dfb51"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.51848184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} {"msg":"Activated sub_volume a514ce27-8f9b-485e-8f62-f705b24836dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:25.518493594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 463 len 20480 data: 9 7 9 11 12 02/30 Read block 219 len 36864 03/30 Read block 455 len 40960 04/30 Write block 347 len 4096 data: 11 05/30 Write block 401 len 32768 data: 13 14 10 8 8 7 7 8 06/30 Read block 189 len 4096 07/30 Read block 439 len 40960 08/30 Write block 342 len 20480 data: 12 14 15 13 16 09/30 Flush 10/30 Write block 271 len 12288 data: 14 18 18 11/30 Write block 206 len 24576 data: 6 6 8 8 6 6 12/30 Write block 555 len 16384 data: 13 15 13 14 13/30 Write block 127 len 8192 data: 8 10 14/30 Write block 408 len 4096 data: 9 15/30 Write block 373 len 4096 data: 11 16/30 Read block 154 len 20480 17/30 Read block 331 len 12288 18/30 Read block 305 len 16384 19/30 Write block 561 len 4096 data: 13 20/30 Read block 459 len 36864 21/30 Read block 572 len 20480 22/30 Read block 500 len 20480 23/30 Read block 384 len 40960 24/30 Flush 25/30 Write block 531 len 32768 data: 12 13 15 13 11 11 11 12 26/30 Read block 526 len 12288 27/30 Write block 106 len 4096 data: 11 28/30 Read block 488 len 20480 29/30 Read block 335 len 28672 30/30 Read block 23 len 16384 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 25 26 Done 30 29 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:25.529797392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1739} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 F F F 1 020-039 117 117 117 193 193 193 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 117 117 117 193 193 193 F F F 5 100-119 114 114 114 189 189 189 T T F <--- 6 120-139 120 120 118 196 196 194 F F F <--- 7 140-159 118 118 118 194 194 194 F F F 8 160-179 118 118 118 194 194 194 F F F 9 180-199 118 118 118 194 194 194 F F F 10 200-219 120 120 118 196 196 194 F F F <--- 11 220-239 118 118 118 194 194 194 F F F 12 240-259 117 117 117 193 193 193 F F F 13 260-279 120 120 117 196 196 193 F F F <--- 14 280-299 118 118 118 194 194 194 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 120 120 118 195 195 194 F F T <--- 18 360-379 120 120 118 196 196 194 F F F <--- 19 380-399 117 117 117 193 193 193 F F F 20 400-419 120 120 117 196 196 193 F F T <--- 21 420-439 117 117 117 193 193 193 F F F 22 440-459 118 118 118 194 194 194 F F F 23 460-479 120 120 118 195 195 194 F F T <--- 24 480-499 117 117 117 193 193 193 F F F 25 500-519 118 118 118 194 194 194 F F F 26 520-539 117 117 117 193 193 193 T T F <--- 27 540-559 120 120 117 196 196 193 F F F <--- 28 560-579 120 120 116 196 196 192 F F F <--- 29 580-599 116 116 116 192 192 192 F F F Max gen: 120, Max flush: 196 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 F F F 1 020-039 117 117 117 193 193 193 F F F 2 040-059 114 114 114 189 189 189 F F F 3 060-079 110 110 110 185 185 185 F F F 4 080-099 117 117 117 193 193 193 F F F 5 100-119 121 121 121 197 197 197 F F F 6 120-139 120 120 120 196 196 196 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 118 118 118 194 194 194 F F F 9 180-199 118 118 118 194 194 194 F F F 10 200-219 120 120 120 196 196 196 F F F 11 220-239 118 118 118 194 194 194 F F F 12 240-259 117 117 117 193 193 193 F F F 13 260-279 120 120 120 196 196 196 F F F 14 280-299 118 118 118 194 194 194 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 F F F 17 340-359 120 120 120 195 195 195 F F F 18 360-379 120 120 120 196 196 196 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 120 120 120 196 196 196 F F F 21 420-439 117 117 117 193 193 193 F F F 22 440-459 118 118 118 194 194 194 F F F 23 460-479 120 120 120 195 195 195 F F F 24 480-499 117 117 117 193 193 193 F F F 25 500-519 118 118 118 194 194 194 F F F 26 520-539 121 121 121 197 197 197 F F F 27 540-559 120 120 120 196 196 196 F F F 28 560-579 120 120 120 196 196 196 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 121, Max flush: 197 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.858019827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870055143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870379089Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870417671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870433896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870451951Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"Crucible 63613c7e-2e43-4edb-a12f-50e59e63f8bb has session id: a809f727-2579-47fe-86fc-8f81de42fbab","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870794163Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Upstairs opts: Upstairs UUID: 63613c7e-2e43-4edb-a12f-50e59e63f8bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870823212Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Crucible stats registered with UUID: 63613c7e-2e43-4edb-a12f-50e59e63f8bb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.870838486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:39.87085173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87102061Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871056701Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871072566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871172376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87118727Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"63613c7e-2e43-4edb-a12f-50e59e63f8bb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871205295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871218319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871232534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871246798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871262923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871277187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871291581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"ds_connection connected from Ok(127.0.0.1:64476)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871337795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"ds_connection connected from Ok(127.0.0.1:63562)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87135422Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"ds_connection connected from Ok(127.0.0.1:33723)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.871380458Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.872000062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.872029841Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.872622128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.872959388Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[0]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 197, 196, 194, 194, 194, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.872981865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[0]R generation[0..12]: [118, 117, 114, 110, 117, 121, 120, 118, 118, 118, 120, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87299828Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873015505Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[1]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 197, 196, 194, 194, 194, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87303144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[1]R generation[0..12]: [118, 117, 114, 110, 117, 121, 120, 118, 118, 118, 120, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873047134Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.8730667Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[2]R flush_numbers[0..12]: [194, 193, 189, 185, 193, 197, 196, 194, 194, 194, 196, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873083025Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[2]R generation[0..12]: [118, 117, 114, 110, 117, 121, 120, 118, 118, 118, 120, 118]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.87309893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873115274Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873128568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873142973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873156507Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873176153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"":"downstairs","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873190187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873202881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"0","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873216975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"1","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873230719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"client":"2","":"downstairs","session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873245083Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873258397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"63613c7e-2e43-4edb-a12f-50e59e63f8bb is now active with session: a809f727-2579-47fe-86fc-8f81de42fbab","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2026-03-17T14:26:39.873278943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873292207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748,"session_id":"a809f727-2579-47fe-86fc-8f81de42fbab"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873305561Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} {"msg":"Activated sub_volume 63613c7e-2e43-4edb-a12f-50e59e63f8bb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:39.873316955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 16384 data: 11 9 10 9 02/30 Read block 432 len 28672 03/30 Read block 188 len 32768 04/30 Read block 203 len 4096 05/30 Write block 84 len 12288 data: 9 8 9 06/30 Write block 236 len 40960 data: 14 16 16 17 15 16 12 10 10 9 07/30 Write block 416 len 36864 data: 9 7 6 7 8 9 9 11 11 08/30 Write block 162 len 12288 data: 5 7 7 09/30 Read block 305 len 16384 10/30 Write block 503 len 12288 data: 11 15 18 11/30 Write block 499 len 28672 data: 6 10 9 7 12 16 19 12/30 Write block 374 len 24576 data: 10 9 10 7 9 9 13/30 Write block 338 len 16384 data: 7 10 9 11 14/30 Read block 38 len 24576 15/30 Read block 136 len 20480 16/30 Write block 47 len 36864 data: 15 16 13 12 9 10 10 10 10 17/30 Read block 408 len 36864 18/30 Write block 252 len 28672 data: 9 9 13 14 13 13 12 19/30 Read block 19 len 16384 20/30 Read block 113 len 8192 21/30 Write block 205 len 28672 data: 7 7 7 9 9 7 7 22/30 Read block 128 len 28672 23/30 Read block 462 len 28672 24/30 Read block 369 len 36864 25/30 Read block 181 len 32768 26/30 Read block 118 len 4096 27/30 Read block 102 len 28672 28/30 Write block 30 len 8192 data: 9 8 29/30 Read block 43 len 8192 30/30 Write block 429 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:39.885087132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1748} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 F F F 1 020-039 117 117 117 193 193 193 T F T <--- 2 040-059 114 114 114 189 189 189 T F T <--- 3 060-079 110 110 110 185 185 185 T F T <--- 4 080-099 117 117 117 193 193 193 T F T <--- 5 100-119 121 121 121 197 197 197 F F F 6 120-139 120 120 120 196 196 196 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 118 118 118 194 194 194 T F T <--- 9 180-199 118 118 118 194 194 194 F F F 10 200-219 120 120 120 196 196 196 T F T <--- 11 220-239 118 118 118 194 194 194 T F T <--- 12 240-259 117 117 117 193 193 193 T F T <--- 13 260-279 120 120 120 196 196 196 F F F 14 280-299 118 118 118 194 194 194 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 115 115 115 190 190 190 T F T <--- 17 340-359 120 120 120 195 195 195 T F T <--- 18 360-379 120 120 120 196 196 196 T F T <--- 19 380-399 117 117 117 193 193 193 F F F 20 400-419 120 120 120 196 196 196 T F T <--- 21 420-439 117 117 117 193 193 193 T F T <--- 22 440-459 118 118 118 194 194 194 F F F 23 460-479 120 120 120 195 195 195 F F F 24 480-499 117 117 117 193 193 193 T F T <--- 25 500-519 118 118 118 194 194 194 T F T <--- 26 520-539 121 121 121 197 197 197 F F F 27 540-559 120 120 120 196 196 196 F F F 28 560-579 120 120 120 196 196 196 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 121, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 122 122 122 198 198 198 F F F 3 060-079 122 122 122 198 198 198 F F F 4 080-099 122 122 122 198 198 198 F F F 5 100-119 121 121 121 197 197 197 F F F 6 120-139 120 120 120 196 196 196 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 122 122 122 198 198 198 F F F 9 180-199 118 118 118 194 194 194 F F F 10 200-219 122 122 122 198 198 198 F F F 11 220-239 122 122 122 198 198 198 F F F 12 240-259 122 122 122 198 198 198 F F F 13 260-279 120 120 120 196 196 196 F F F 14 280-299 118 118 118 194 194 194 F F F 15 300-319 113 113 113 188 188 188 F F F 16 320-339 122 122 122 198 198 198 F F F 17 340-359 122 122 122 198 198 198 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 122 122 122 198 198 198 F F F 22 440-459 118 118 118 194 194 194 F F F 23 460-479 120 120 120 195 195 195 F F F 24 480-499 122 122 122 198 198 198 F F F 25 500-519 122 122 122 198 198 198 F F F 26 520-539 121 121 121 197 197 197 F F F 27 540-559 120 120 120 196 196 196 F F F 28 560-579 120 120 120 196 196 196 F F F 29 580-599 116 116 116 192 192 192 F F F Max gen: 122, Max flush: 198 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.225980823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.237813469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238173786Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238214298Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238226862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238244697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"Crucible efd1759a-9e48-4338-91d3-8bcad122fd23 has session id: 665dacf6-2089-44a6-87a0-2b00e4b0051c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238595652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Upstairs opts: Upstairs UUID: efd1759a-9e48-4338-91d3-8bcad122fd23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238629942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Crucible stats registered with UUID: efd1759a-9e48-4338-91d3-8bcad122fd23","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238645307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:54.238659151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.23882527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238855609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.238870734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239008555Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.23902444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"efd1759a-9e48-4338-91d3-8bcad122fd23 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239041695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239054589Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239068833Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239082787Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239096441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239115477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239129711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37053)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239216267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44180)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239284207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43170)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239311755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.239903352Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.23993049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.24009813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240449874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[0]R flush_numbers[0..12]: [194, 198, 198, 198, 198, 197, 196, 194, 198, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240474022Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[0]R generation[0..12]: [118, 122, 122, 122, 122, 121, 120, 118, 122, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240489836Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240506411Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[1]R flush_numbers[0..12]: [194, 198, 198, 198, 198, 197, 196, 194, 198, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240521976Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[1]R generation[0..12]: [118, 122, 122, 122, 122, 121, 120, 118, 122, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240549114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240565459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[2]R flush_numbers[0..12]: [194, 198, 198, 198, 198, 197, 196, 194, 198, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240581454Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[2]R generation[0..12]: [118, 122, 122, 122, 122, 121, 120, 118, 122, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240597128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240613243Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Max found gen is 123","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240626617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Generation requested: 124 >= found:123","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240640691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid"Wait for a query_work_queue command to finish before sending IO :1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240663738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240677172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"":"downstairs","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240691607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.2407044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"0","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240718765Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"1","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240732979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"client":"2","":"downstairs","session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240747153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240760497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"efd1759a-9e48-4338-91d3-8bcad122fd23 is now active with session: 665dacf6-2089-44a6-87a0-2b00e4b0051c","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240778632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240791696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757,"session_id":"665dacf6-2089-44a6-87a0-2b00e4b0051c"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.24080509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} {"msg":"Activated sub_volume efd1759a-9e48-4338-91d3-8bcad122fd23","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:26:54.240816474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 16384 data: 10 9 9 7 02/30 Read block 148 len 20480 03/30 Read block 92 len 16384 04/30 Write block 501 len 24576 data: 10 8 13 17 20 18 05/30 Read block 474 len 28672 06/30 Write block 272 len 20480 data: 19 19 13 11 9 07/30 Write block 557 len 40960 data: 14 15 13 13 14 13 13 13 9 10 08/30 Write block 479 len 32768 data: 10 9 9 8 7 6 7 7 09/30 Write block 534 len 20480 data: 14 12 12 12 13 10/30 Flush 11/30 Write block 542 len 36864 data: 10 9 10 9 9 9 10 9 10 12/30 Read block 216 len 16384 13/30 Read block 82 len 28672 14/30 Write block 162 len 36864 data: 6 8 8 7 6 6 6 4 3 15/30 Write block 435 len 24576 data: 11 10 8 6 5 3 16/30 Read block 439 len 24576 17/30 Write block 97 len 36864 data: 10 9 9 12 11 14 12 12 13 18/30 Write block 574 len 36864 data: 5 5 5 7 7 7 12 12 11 19/30 Read block 278 len 24576 20/30 Write block 175 len 20480 data: 12 14 11 9 11 21/30 Read block 513 len 8192 22/30 Flush 23/30 Write block 67 len 24576 data: 9 12 12 11 9 9 24/30 Read block 443 len 24576 25/30 Write block 7 len 8192 data: 7 8 26/30 Read block 93 len 36864 27/30 Read block 374 len 40960 28/30 Read block 251 len 16384 29/30 Write block 476 len 40960 data: 9 9 9 11 10 10 9 8 7 8 30/30 Write block 56 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:26:54.254340873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1757} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 194 194 194 T F T <--- 1 020-039 122 122 122 198 198 198 F F F 2 040-059 122 122 122 198 198 198 T F T <--- 3 060-079 122 122 122 198 198 198 T F T <--- 4 080-099 124 122 124 200 198 200 F F F <--- 5 100-119 124 121 124 200 197 200 F F F <--- 6 120-139 120 120 120 196 196 196 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 124 122 124 200 198 200 F F F <--- 9 180-199 118 118 118 194 194 194 F F F 10 200-219 122 122 122 198 198 198 F F F 11 220-239 122 122 122 198 198 198 F F F 12 240-259 122 122 122 198 198 198 F F F 13 260-279 124 120 124 199 196 199 F F F <--- 14 280-299 118 118 118 194 194 194 F F F 15 300-319 124 113 124 199 188 199 F F F <--- 16 320-339 122 122 122 198 198 198 F F F 17 340-359 122 122 122 198 198 198 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 122 124 200 198 200 F F F <--- 22 440-459 124 118 124 200 194 200 F F F <--- 23 460-479 124 120 124 199 195 199 T F T <--- 24 480-499 124 122 124 199 198 199 T F T <--- 25 500-519 124 122 124 199 198 199 F F F <--- 26 520-539 124 121 124 199 197 199 F F F <--- 27 540-559 124 120 124 200 196 200 F F F <--- 28 560-579 124 120 124 200 196 200 F F F <--- 29 580-599 124 116 124 200 192 200 F F F <--- Max gen: 124, Max flush: 200 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 201 201 201 F F F 1 020-039 122 122 122 198 198 198 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 124 124 124 200 200 200 F F F 5 100-119 124 124 124 200 200 200 F F F 6 120-139 120 120 120 196 196 196 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 124 124 124 200 200 200 F F F 9 180-199 118 118 118 194 194 194 F F F 10 200-219 122 122 122 198 198 198 F F F 11 220-239 122 122 122 198 198 198 F F F 12 240-259 122 122 122 198 198 198 F F F 13 260-279 124 124 124 199 199 199 F F F 14 280-299 118 118 118 194 194 194 F F F 15 300-319 124 124 124 199 199 199 F F F 16 320-339 122 122 122 198 198 198 F F F 17 340-359 122 122 122 198 198 198 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 124 124 124 200 200 200 F F F 23 460-479 125 125 125 201 201 201 F F F 24 480-499 125 125 125 201 201 201 F F F 25 500-519 124 124 124 199 199 199 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 124 124 124 200 200 200 F F F 28 560-579 124 124 124 200 200 200 F F F 29 580-599 124 124 124 200 200 200 F F F Max gen: 125, Max flush: 201 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.597397308Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:08.608114551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.608167487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.61861617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.618890862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.618928904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.618941247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.618959263Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"Crucible 5a4c4ef7-07f4-4fd7-8316-5c425c836f2b has session id: 37f78906-ba76-4314-a8f9-495491c39963","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619296303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Upstairs opts: Upstairs UUID: 5a4c4ef7-07f4-4fd7-8316-5c425c836f2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619361673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Crucible stats registered with UUID: 5a4c4ef7-07f4-4fd7-8316-5c425c836f2b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619378448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:08.619391992Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619542556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619569314Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619584299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619728492Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619743706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"5a4c4ef7-07f4-4fd7-8316-5c425c836f2b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619765723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619778647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619795112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619808966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.61982267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619837074Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619851559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"ds_connection connected from Ok(127.0.0.1:46884)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619892751Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"ds_connection connected from Ok(127.0.0.1:36580)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619914697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"ds_connection connected from Ok(127.0.0.1:57821)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.619930202Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.620716456Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.620744715Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.62076181Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621125148Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[0]R flush_numbers[0..12]: [201, 198, 201, 201, 200, 200, 196, 194, 200, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621147635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[0]R generation[0..12]: [125, 122, 125, 125, 124, 124, 120, 118, 124, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621163549Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621182555Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[1]R flush_numbers[0..12]: [201, 198, 201, 201, 200, 200, 196, 194, 200, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.62119845Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[1]R generation[0..12]: [125, 122, 125, 125, 124, 124, 120, 118, 124, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621214225Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621230799Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[2]R flush_numbers[0..12]: [201, 198, 201, 201, 200, 200, 196, 194, 200, 194, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621246694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[2]R generation[0..12]: [125, 122, 125, 125, 124, 124, 120, 118, 124, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621262239Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621278604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621292568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621312484Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621326158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621339552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"":"downstairs","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621353686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.62136612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"0","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621380024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"1","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621393998Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"client":"2","":"downstairs","session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-17T14:27:08.621410063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621426168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"5a4c4ef7-07f4-4fd7-8316-5c425c836f2b is now active with session: 37f78906-ba76-4314-a8f9-495491c39963","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621441472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621454346Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766,"session_id":"37f78906-ba76-4314-a8f9-495491c39963"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.62146779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} {"msg":"Activated sub_volume 5a4c4ef7-07f4-4fd7-8316-5c425c836f2b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:08.621479404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 494 len 36864 data: 11 9 6 6 6 7 11 11 9 03/30 Read block 81 len 36864 04/30 Write block 317 len 28672 data: 10 9 10 11 11 10 9 05/30 Flush 06/30 Flush 07/30 Write block 191 len 36864 data: 10 8 7 7 7 13 15 14 13 08/30 Read block 89 len 12288 09/30 Write block 447 len 28672 data: 5 6 6 9 11 10 10 10/30 Write block 284 len 8192 data: 11 12 11/30 Write block 35 len 16384 data: 10 10 9 9 12/30 Write block 1 len 24576 data: 4 4 4 4 6 7 13/30 Write block 228 len 12288 data: 10 10 11 14/30 Write block 120 len 24576 data: 7 8 10 7 8 7 15/30 Write block 0 len 8192 data: 3 5 16/30 Read block 49 len 12288 17/30 Read block 13 len 12288 18/30 Write block 443 len 40960 data: 7 11 12 11 6 7 7 10 12 11 19/30 Flush 20/30 Flush 21/30 Write block 524 len 20480 data: 6 6 6 10 10 22/30 Write block 293 len 32768 data: 10 13 11 12 10 11 12 12 23/30 Read block 315 len 8192 24/30 Write block 580 len 24576 data: 13 13 12 11 9 9 25/30 Read block 467 len 16384 26/30 Read block 22 len 36864 27/30 Write block 577 len 16384 data: 8 8 8 14 28/30 Read block 437 len 32768 29/30 Write block 474 len 12288 data: 7 8 10 30/30 Write block 246 len 8192 data: 10 10 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:08.633880869Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1766} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 126 126 201 205 205 F F F <--- 1 020-039 122 126 126 198 205 205 F F F <--- 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 124 124 124 200 200 200 F F F 5 100-119 124 124 124 200 200 200 F F F 6 120-139 120 126 126 196 205 205 F F F <--- 7 140-159 118 118 118 194 194 194 F F F 8 160-179 124 124 124 200 200 200 F F F 9 180-199 118 126 126 194 205 205 F F F <--- 10 200-219 122 122 122 198 198 198 F F F 11 220-239 122 126 126 198 205 205 F F F <--- 12 240-259 122 122 122 198 198 198 F T T <--- 13 260-279 124 124 124 199 199 199 F F F 14 280-299 118 126 126 194 205 205 F T T <--- 15 300-319 124 126 126 199 203 203 F T T <--- 16 320-339 122 126 126 198 203 203 F F F <--- 17 340-359 122 122 122 198 198 198 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 124 126 126 200 205 205 F F F <--- 23 460-479 125 125 125 201 201 201 F T T <--- 24 480-499 125 126 126 201 203 203 F F F <--- 25 500-519 124 126 126 199 203 203 F F F <--- 26 520-539 124 124 124 199 199 199 F T T <--- 27 540-559 124 124 124 200 200 200 F F F 28 560-579 124 124 124 200 200 200 F T T <--- 29 580-599 124 124 124 200 200 200 F T T <--- Max gen: 126, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 205 205 205 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 124 124 124 200 200 200 F F F 5 100-119 124 124 124 200 200 200 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 118 118 118 194 194 194 F F F 8 160-179 124 124 124 200 200 200 F F F 9 180-199 126 126 126 205 205 205 F F F 10 200-219 122 122 122 198 198 198 F F F 11 220-239 126 126 126 205 205 205 F F F 12 240-259 127 127 127 206 206 206 F F F 13 260-279 124 124 124 199 199 199 F F F 14 280-299 127 127 127 206 206 206 F F F 15 300-319 127 127 127 206 206 206 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 122 122 122 198 198 198 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 126 126 126 205 205 205 F F F 23 460-479 127 127 127 206 206 206 F F F 24 480-499 126 126 126 203 203 203 F F F 25 500-519 126 126 126 203 203 203 F F F 26 520-539 127 127 127 206 206 206 F F F 27 540-559 124 124 124 200 200 200 F F F 28 560-579 127 127 127 206 206 206 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 127, Max flush: 206 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.969859064Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.98155696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.981871814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.981909815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.981921909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.981939944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"Crucible 711eb9a2-5f54-4f3e-b329-65583711f1be has session id: 3201a953-79f0-4111-a9a9-91c867d0733a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982287647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Upstairs opts: Upstairs UUID: 711eb9a2-5f54-4f3e-b329-65583711f1be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982323038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Crucible stats registered with UUID: 711eb9a2-5f54-4f3e-b329-65583711f1be","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982341223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:22.982354537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982524638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982548355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.98256296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982720496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982737121Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"711eb9a2-5f54-4f3e-b329-65583711f1be active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982760548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982773662Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982787716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982801881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982816015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982830699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.982845174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64708)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.98293466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56993)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.983007882Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34245)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.983027928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.983681983Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.983723725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984387873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984742559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 201, 201, 200, 200, 205, 194, 200, 205, 198, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984765486Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[0]R generation[0..12]: [126, 126, 125, 125, 124, 124, 126, 118, 124, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984781681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984798276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 201, 201, 200, 200, 205, 194, 200, 205, 198, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.98481415Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[1]R generation[0..12]: [126, 126, 125, 125, 124, 124, 126, 118, 124, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984830035Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.9848468Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 201, 201, 200, 200, 205, 194, 200, 205, 198, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984862775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[2]R generation[0..12]: [126, 126, 125, 125, 124, 124, 126, 118, 124, 126, 122, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984878199Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984894714Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984911059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:27:22.984925143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984944189Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984957923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"":"downstairs","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984971897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984984231Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"0","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.984998495Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"1","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.985012509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"client":"2","":"downstairs","session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.985026814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:27:22.985040638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"711eb9a2-5f54-4f3e-b329-65583711f1be is now active with session: 3201a953-79f0-4111-a9a9-91c867d0733a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.985056462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.985069396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775,"session_id":"3201a953-79f0-4111-a9a9-91c867d0733a"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.98508282Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} {"msg":"Activated sub_volume 711eb9a2-5f54-4f3e-b329-65583711f1be","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:22.985094804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 565 len 16384 data: 10 11 11 11 03/30 Write block 572 len 32768 data: 5 5 6 6 6 9 9 9 04/30 Write block 342 len 16384 data: 13 15 16 14 05/30 Write block 213 len 28672 data: 6 6 6 9 8 9 10 06/30 Read block 117 len 12288 07/30 Flush 08/30 Write block 296 len 20480 data: 13 11 12 13 13 09/30 Read block 454 len 32768 10/30 Flush 11/30 Write block 148 len 8192 data: 10 10 12/30 Read block 474 len 20480 13/30 Write block 516 len 28672 data: 11 13 11 8 7 6 7 14/30 Read block 438 len 36864 15/30 Write block 298 len 16384 data: 13 14 14 10 16/30 Read block 229 len 36864 17/30 Read block 291 len 36864 18/30 Write block 243 len 32768 data: 11 11 10 11 11 13 14 14 19/30 Read block 154 len 28672 20/30 Read block 113 len 12288 21/30 Write block 147 len 8192 data: 9 11 22/30 Read block 338 len 24576 23/30 Read block 592 len 8192 24/30 Read block 139 len 32768 25/30 Read block 549 len 40960 26/30 Flush 27/30 Write block 119 len 4096 data: 8 28/30 Read block 43 len 32768 29/30 Write block 95 len 40960 data: 8 8 11 10 10 13 12 15 13 13 30/30 Write block 107 len 24576 data: 12 8 10 12 10 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 23 27 Done 21 21 0 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:22.997295989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1775} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 205 205 205 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 124 124 124 200 200 200 T T F <--- 5 100-119 124 124 124 200 200 200 T T F <--- 6 120-139 126 126 126 205 205 205 F F F 7 140-159 128 128 118 210 210 194 F F F <--- 8 160-179 124 124 124 200 200 200 F F F 9 180-199 126 126 126 205 205 205 F F F 10 200-219 128 128 128 208 208 208 F F F 11 220-239 126 126 126 205 205 205 F F F 12 240-259 128 128 127 210 210 206 F F F <--- 13 260-279 124 124 124 199 199 199 F F F 14 280-299 128 128 127 210 210 206 F F F <--- 15 300-319 128 128 127 210 210 206 F F F <--- 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 126 126 126 205 205 205 F F F 23 460-479 127 127 127 206 206 206 F F F 24 480-499 126 126 126 203 203 203 F F F 25 500-519 128 128 126 210 210 203 F F F <--- 26 520-539 128 128 127 210 210 206 F F F <--- 27 540-559 124 124 124 200 200 200 F F F 28 560-579 128 128 128 208 208 208 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 128, Max flush: 210 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 205 205 205 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 129 129 129 211 211 211 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 128 128 128 210 210 210 F F F 8 160-179 124 124 124 200 200 200 F F F 9 180-199 126 126 126 205 205 205 F F F 10 200-219 128 128 128 208 208 208 F F F 11 220-239 126 126 126 205 205 205 F F F 12 240-259 128 128 128 210 210 210 F F F 13 260-279 124 124 124 199 199 199 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 128 128 128 210 210 210 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 122 122 122 198 198 198 F F F 19 380-399 117 117 117 193 193 193 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 126 126 126 205 205 205 F F F 23 460-479 127 127 127 206 206 206 F F F 24 480-499 126 126 126 203 203 203 F F F 25 500-519 128 128 128 210 210 210 F F F 26 520-539 128 128 128 210 210 210 F F F 27 540-559 124 124 124 200 200 200 F F F 28 560-579 128 128 128 208 208 208 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 129, Max flush: 211 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.313375836Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325245413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325553725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325591966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325607381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325625056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"Crucible cf5eb9ea-7e83-42c3-a127-b2498cb1d22d has session id: f47025b0-1480-4136-829c-efb81f0c8569","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325965317Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Upstairs opts: Upstairs UUID: cf5eb9ea-7e83-42c3-a127-b2498cb1d22d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.325991635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Crucible stats registered with UUID: cf5eb9ea-7e83-42c3-a127-b2498cb1d22d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.32600663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:37.326019944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326191585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326216212Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326230917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326397506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326421683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"cf5eb9ea-7e83-42c3-a127-b2498cb1d22d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326433877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326446651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326461105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.32647847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326508179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326546751Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326566587Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"ds_connection connected from Ok(127.0.0.1:37244)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326581531Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"ds_connection connected from Ok(127.0.0.1:41331)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326596395Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"ds_connection connected from Ok(127.0.0.1:35431)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.326672848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327313999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327341508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327358082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.32775108Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 201, 201, 211, 211, 205, 210, 200, 205, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327773426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[0]R generation[0..12]: [126, 126, 125, 125, 129, 129, 126, 128, 124, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327789451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327805806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 201, 201, 211, 211, 205, 210, 200, 205, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327821681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[1]R generation[0..12]: [126, 126, 125, 125, 129, 129, 126, 128, 124, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327840506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327857161Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 201, 201, 211, 211, 205, 210, 200, 205, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327872656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[2]R generation[0..12]: [126, 126, 125, 125, 129, 129, 126, 128, 124, 126, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.32788797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327903935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327917399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327937475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327950869Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327964093Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"":"downstairs","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327977887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.327990571Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"0","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328004815Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"1","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328018909Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"client":"2","":"downstairs","session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328033284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328048408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"cf5eb9ea-7e83-42c3-a127-b2498cb1d22d is now active with session: f47025b0-1480-4136-829c-efb81f0c8569","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328066223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328079247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784,"session_id":"f47025b0-1480-4136-829c-efb81f0c8569"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328092691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} {"msg":"Activated sub_volume cf5eb9ea-7e83-42c3-a127-b2498cb1d22d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:37.328104255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 24576 data: 5 7 8 8 9 10 02/30 Write block 549 len 28672 data: 10 11 11 11 12 14 14 03/30 Write block 563 len 20480 data: 14 14 11 12 12 04/30 Read block 397 len 8192 05/30 Write block 384 len 20480 data: 6 12 9 11 9 06/30 Read block 102 len 4096 07/30 Read block 516 len 36864 08/30 Write block 315 len 20480 data: 8 10 11 10 11 09/30 Write block 255 len 12288 data: 15 14 14 10/30 Write block 166 len 28672 data: 7 7 7 5 4 5 7 11/30 Read block 287 len 36864 12/30 Write block 511 len 8192 data: 14 13 13/30 Read block 86 len 8192 14/30 Read block 527 len 16384 15/30 Read block 577 len 24576 16/30 Write block 456 len 28672 data: 7 7 6 6 7 8 10 17/30 Read block 529 len 20480 18/30 Write block 374 len 20480 data: 11 10 11 8 10 19/30 Read block 484 len 28672 20/30 Read block 512 len 20480 21/30 Read block 197 len 40960 22/30 Write block 196 len 8192 data: 14 16 23/30 Read block 464 len 16384 24/30 Write block 156 len 36864 data: 8 8 8 8 6 5 7 9 9 25/30 Flush 26/30 Write block 382 len 32768 data: 10 7 7 13 10 12 10 9 27/30 Write block 560 len 16384 data: 14 15 14 15 28/30 Write block 259 len 20480 data: 12 12 11 12 11 29/30 Read block 101 len 28672 30/30 Read block 217 len 36864 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 2 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:37.340319754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 126 212 212 205 F F T <--- 1 020-039 126 126 126 205 205 205 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 129 129 129 211 211 211 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 130 130 128 212 212 210 F F F <--- 8 160-179 130 130 124 212 212 200 F F F <--- 9 180-199 130 130 126 212 212 205 F F F <--- 10 200-219 128 128 128 208 208 208 F F F 11 220-239 126 126 126 205 205 205 F F F 12 240-259 130 130 128 212 212 210 T T F <--- 13 260-279 124 124 124 199 199 199 T T F <--- 14 280-299 128 128 128 210 210 210 F F F 15 300-319 130 130 128 212 212 210 F F F <--- 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 130 130 122 212 212 198 F F F <--- 19 380-399 130 130 117 212 212 193 T T F <--- 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 130 130 126 212 212 205 F F F <--- 23 460-479 130 130 127 212 212 206 F F F <--- 24 480-499 126 126 126 203 203 203 F F F 25 500-519 130 130 128 212 212 210 F F F <--- 26 520-539 128 128 128 210 210 210 F F F 27 540-559 130 130 124 212 212 200 F F T <--- 28 560-579 130 130 128 212 212 208 T T F <--- 29 580-599 127 127 127 206 206 206 F F F Max gen: 130, Max flush: 212 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 125 125 125 201 201 201 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 129 129 129 211 211 211 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 130 130 130 212 212 212 F F F 8 160-179 130 130 130 212 212 212 F F F 9 180-199 130 130 130 212 212 212 F F F 10 200-219 128 128 128 208 208 208 F F F 11 220-239 126 126 126 205 205 205 F F F 12 240-259 131 131 131 213 213 213 F F F 13 260-279 131 131 131 213 213 213 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 130 130 130 212 212 212 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 130 130 130 212 212 212 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 130 130 130 212 212 212 F F F 23 460-479 130 130 130 212 212 212 F F F 24 480-499 126 126 126 203 203 203 F F F 25 500-519 130 130 130 212 212 212 F F F 26 520-539 128 128 128 210 210 210 F F F 27 540-559 130 130 130 212 212 212 F F F 28 560-579 131 131 131 213 213 213 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 131, Max flush: 213 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.671435881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683132556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683441258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683479789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683492423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683510098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"Crucible cb39b76a-17be-475d-8c52-bd3c22f13a37 has session id: 00f1feae-6fc7-4fe3-85a6-b5d530a48953","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683841567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Upstairs opts: Upstairs UUID: cb39b76a-17be-475d-8c52-bd3c22f13a37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683882719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Crucible stats registered with UUID: cb39b76a-17be-475d-8c52-bd3c22f13a37","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.683898954Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:51.683915289Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684088331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684122881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684138646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684276767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684309307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"cb39b76a-17be-475d-8c52-bd3c22f13a37 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.68432197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684335654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684349989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684363953Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684377797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684404785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684419159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"ds_connection connected from Ok(127.0.0.1:41113)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684459641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"ds_connection connected from Ok(127.0.0.1:59172)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684580607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"ds_connection connected from Ok(127.0.0.1:48636)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.684608406Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685240714Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685309284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685373204Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685702522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[0]R flush_numbers[0..12]: [212, 205, 201, 201, 211, 211, 205, 212, 212, 212, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685726519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[0]R generation[0..12]: [130, 126, 125, 125, 129, 129, 126, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685742744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685759219Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[1]R flush_numbers[0..12]: [212, 205, 201, 201, 211, 211, 205, 212, 212, 212, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685774973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[1]R generation[0..12]: [130, 126, 125, 125, 129, 129, 126, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685790848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685807413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[2]R flush_numbers[0..12]: [212, 205, 201, 201, 211, 211, 205, 212, 212, 212, 208, 205]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685823258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[2]R generation[0..12]: [130, 126, 125, 125, 129, 129, 126, 130, 130, 130, 128, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685838942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685855557Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685868821Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:27:51.685885366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685905362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685918696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"":"downstairs","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.68593321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685945954Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"0","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685960108Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"1","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685974063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"client":"2","":"downstairs","session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.685988047Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.686001491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"cb39b76a-17be-475d-8c52-bd3c22f13a37 is now active with session: 00f1feae-6fc7-4fe3-85a6-b5d530a48953","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.686019636Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.68603251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793,"session_id":"00f1feae-6fc7-4fe3-85a6-b5d530a48953"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.686045924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} {"msg":"Activated sub_volume cb39b76a-17be-475d-8c52-bd3c22f13a37","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:27:51.686057457Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 561 len 28672 data: 16 15 16 15 12 13 13 02/30 Write block 46 len 16384 data: 12 16 17 14 03/30 Read block 386 len 8192 04/30 Write block 248 len 16384 data: 14 15 15 10 05/30 Read block 92 len 36864 06/30 Write block 365 len 28672 data: 8 5 4 4 5 5 7 07/30 Read block 37 len 4096 08/30 Read block 437 len 12288 09/30 Write block 171 len 4096 data: 6 10/30 Read block 107 len 4096 11/30 Write block 111 len 4096 data: 11 12/30 Write block 233 len 16384 data: 13 14 16 15 13/30 Read block 324 len 40960 14/30 Read block 455 len 12288 15/30 Write block 555 len 12288 data: 15 16 15 16/30 Read block 560 len 40960 17/30 Read block 394 len 36864 18/30 Read block 344 len 32768 19/30 Read block 200 len 40960 20/30 Write block 464 len 8192 data: 8 10 21/30 Read block 490 len 36864 22/30 Write block 187 len 32768 data: 9 9 9 9 11 9 8 8 23/30 Read block 167 len 36864 24/30 Write block 541 len 16384 data: 12 11 10 11 25/30 Write block 507 len 36864 data: 19 17 15 15 15 14 13 12 11 26/30 Flush 27/30 Write block 541 len 4096 data: 13 28/30 Write block 370 len 40960 data: 6 8 8 12 12 11 12 9 11 10 29/30 Read block 408 len 28672 30/30 Read block 78 len 32768 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 16 0 16 Done 30 14 30 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:27:51.698574407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1793} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 125 132 214 201 214 F T F <--- 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 132 129 132 214 211 214 F T F <--- 6 120-139 126 126 126 205 205 205 F F F 7 140-159 130 130 130 212 212 212 F F F 8 160-179 132 130 132 214 212 214 F T F <--- 9 180-199 132 130 132 214 212 214 F F F <--- 10 200-219 128 128 128 208 208 208 F F F 11 220-239 132 126 132 214 205 214 F T F <--- 12 240-259 132 131 132 214 213 214 F T F <--- 13 260-279 131 131 131 213 213 213 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 130 130 130 212 212 212 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 132 130 132 214 212 214 T T T <--- 19 380-399 131 131 131 213 213 213 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 130 130 130 212 212 212 F F F 23 460-479 132 130 132 214 212 214 F F F <--- 24 480-499 126 126 126 203 203 203 F F F 25 500-519 132 130 132 214 212 214 F F F <--- 26 520-539 128 128 128 210 210 210 F F F 27 540-559 132 130 132 214 212 214 T F T <--- 28 560-579 132 131 132 214 213 214 F T F <--- 29 580-599 127 127 127 206 206 206 F F F Max gen: 132, Max flush: 214 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 132 132 132 214 214 214 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 130 130 130 212 212 212 F F F 8 160-179 132 132 132 214 214 214 F F F 9 180-199 132 132 132 214 214 214 F F F 10 200-219 128 128 128 208 208 208 F F F 11 220-239 132 132 132 214 214 214 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 131 131 131 213 213 213 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 130 130 130 212 212 212 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 F F F 18 360-379 133 133 133 215 215 215 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 122 122 122 198 198 198 F F F 21 420-439 124 124 124 200 200 200 F F F 22 440-459 130 130 130 212 212 212 F F F 23 460-479 132 132 132 214 214 214 F F F 24 480-499 126 126 126 203 203 203 F F F 25 500-519 132 132 132 214 214 214 F F F 26 520-539 128 128 128 210 210 210 F F F 27 540-559 133 133 133 215 215 215 F F F 28 560-579 132 132 132 214 214 214 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 133, Max flush: 215 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.027842682Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.039590032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.039906947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.03995035Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.039966905Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.03998558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"Crucible 42c499a5-4b78-4dc2-a996-eccdc887f0c2 has session id: 1d9aa9a3-029e-4c41-9a52-46679fe3f301","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040305545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Upstairs opts: Upstairs UUID: 42c499a5-4b78-4dc2-a996-eccdc887f0c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040374536Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Crucible stats registered with UUID: 42c499a5-4b78-4dc2-a996-eccdc887f0c2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040391671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:06.040404885Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040576426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040602244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040616838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040770374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040794861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"42c499a5-4b78-4dc2-a996-eccdc887f0c2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040807275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040820139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040834813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040852178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040899702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.04092495Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040940275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"ds_connection connected from Ok(127.0.0.1:43867)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.040955919Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"ds_connection connected from Ok(127.0.0.1:63224)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.041108075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"ds_connection connected from Ok(127.0.0.1:44959)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.041135433Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.041707543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.041735622Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.041793479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042156077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[0]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 212, 214, 214, 208, 214]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042178714Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[0]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 130, 132, 132, 128, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042194928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042211273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[1]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 212, 214, 214, 208, 214]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042230769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[1]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 130, 132, 132, 128, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042247324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042263899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[2]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 212, 214, 214, 208, 214]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042279584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[2]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 130, 132, 132, 128, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042295228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042311503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042325107Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042345583Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042359317Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042372751Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"":"downstairs","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042386876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042399539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"0","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042413383Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"1","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042427488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"client":"2","":"downstairs","session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042441892Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042460167Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"42c499a5-4b78-4dc2-a996-eccdc887f0c2 is now active with session: 1d9aa9a3-029e-4c41-9a52-46679fe3f301","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042475762Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042488696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802,"session_id":"1d9aa9a3-029e-4c41-9a52-46679fe3f301"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.04250233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} {"msg":"Activated sub_volume 42c499a5-4b78-4dc2-a996-eccdc887f0c2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:06.042513923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 36864 data: 10 13 16 15 16 14 12 10 8 02/30 Read block 168 len 16384 03/30 Write block 506 len 28672 data: 19 20 18 16 16 16 15 04/30 Read block 536 len 8192 05/30 Write block 303 len 32768 data: 8 9 7 7 7 11 10 10 06/30 Read block 290 len 40960 07/30 Read block 411 len 28672 08/30 Read block 196 len 20480 09/30 Write block 569 len 12288 data: 10 8 7 10/30 Write block 301 len 16384 data: 11 10 9 10 11/30 Read block 18 len 40960 12/30 Read block 555 len 12288 13/30 Write block 213 len 32768 data: 7 7 7 10 9 10 11 10 14/30 Read block 222 len 40960 15/30 Write block 301 len 12288 data: 12 11 10 16/30 Write block 531 len 32768 data: 13 14 16 15 13 13 13 14 17/30 Write block 403 len 8192 data: 11 9 18/30 Read block 448 len 4096 19/30 Write block 266 len 40960 data: 11 15 12 13 13 15 20 20 14 12 20/30 Read block 393 len 8192 21/30 Write block 484 len 16384 data: 8 9 8 10 22/30 Read block 301 len 16384 23/30 Read block 428 len 40960 24/30 Write block 465 len 16384 data: 11 12 13 11 25/30 Write block 156 len 24576 data: 9 9 9 9 7 6 26/30 Write block 414 len 40960 data: 11 8 10 8 7 8 9 10 10 12 27/30 Read block 169 len 28672 28/30 Read block 448 len 16384 29/30 Read block 160 len 28672 30/30 Read block 138 len 40960 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 0 29 Done 29 2 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:06.055797942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1802} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 132 132 132 214 214 214 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 130 130 130 212 212 212 T F T <--- 8 160-179 132 132 132 214 214 214 T F T <--- 9 180-199 132 132 132 214 214 214 F F F 10 200-219 128 128 128 208 208 208 T F T <--- 11 220-239 132 132 132 214 214 214 T F T <--- 12 240-259 132 132 132 214 214 214 F F F 13 260-279 131 131 131 213 213 213 T F T <--- 14 280-299 128 128 128 210 210 210 F F F 15 300-319 130 130 130 212 212 212 T F T <--- 16 320-339 126 126 126 203 203 203 F F F 17 340-359 128 128 128 208 208 208 T T T 18 360-379 133 133 133 215 215 215 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 122 122 122 198 198 198 T F T <--- 21 420-439 124 124 124 200 200 200 T F T <--- 22 440-459 130 130 130 212 212 212 F F F 23 460-479 132 132 132 214 214 214 T F T <--- 24 480-499 126 126 126 203 203 203 T F T <--- 25 500-519 132 132 132 214 214 214 T F T <--- 26 520-539 128 128 128 210 210 210 T F T <--- 27 540-559 133 133 133 215 215 215 F F F 28 560-579 132 132 132 214 214 214 T F T <--- 29 580-599 127 127 127 206 206 206 F F F Max gen: 133, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 132 132 132 214 214 214 F F F 6 120-139 126 126 126 205 205 205 F F F 7 140-159 134 134 134 216 216 216 F F F 8 160-179 134 134 134 216 216 216 F F F 9 180-199 132 132 132 214 214 214 F F F 10 200-219 134 134 134 216 216 216 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 134 134 134 216 216 216 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 134 134 134 216 216 216 F F F 16 320-339 126 126 126 203 203 203 F F F 17 340-359 134 134 134 216 216 216 F F F 18 360-379 133 133 133 215 215 215 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 134 134 134 216 216 216 F F F 21 420-439 134 134 134 216 216 216 F F F 22 440-459 130 130 130 212 212 212 F F F 23 460-479 134 134 134 216 216 216 F F F 24 480-499 134 134 134 216 216 216 F F F 25 500-519 134 134 134 216 216 216 F F F 26 520-539 134 134 134 216 216 216 F F F 27 540-559 133 133 133 215 215 215 F F F 28 560-579 134 134 134 216 216 216 F F F 29 580-599 127 127 127 206 206 206 F F F Max gen: 134, Max flush: 216 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.391719922Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.403479666Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.403791999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.40383054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.403843034Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.403861039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"Crucible d91eb7d6-50ea-4fe5-8be4-3f58179f524a has session id: 762d772e-a40c-4d98-a6f8-4561780bd4eb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404224688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Upstairs opts: Upstairs UUID: d91eb7d6-50ea-4fe5-8be4-3f58179f524a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404260038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Crucible stats registered with UUID: d91eb7d6-50ea-4fe5-8be4-3f58179f524a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404275743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:20.404291878Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404463859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404489307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404504541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404649114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.40466876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"d91eb7d6-50ea-4fe5-8be4-3f58179f524a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404684585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404697519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404711403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404725477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404739581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404753995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404775412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49959)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404796918Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34602)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404812263Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:40745)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.404952255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.405628276Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.405663166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.405800877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406159334Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[0]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 216, 216, 214, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406183741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[0]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 134, 134, 132, 134, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406199926Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406216811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[1]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 216, 216, 214, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406232906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[1]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 134, 134, 132, 134, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406248651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406265156Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[2]R flush_numbers[0..12]: [212, 205, 214, 201, 211, 214, 205, 216, 216, 214, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.40628096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[2]R generation[0..12]: [130, 126, 132, 125, 129, 132, 126, 134, 134, 132, 134, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406296565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.40631292Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Max found gen is 135","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406326574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Generation requested: 136 >= found:135","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406343929Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406363215Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406376429Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"":"downstairs","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406390443Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406403037Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"0","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406416961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"1","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406431305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"client":"2","":"downstairs","session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406445119Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406458373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"d91eb7d6-50ea-4fe5-8be4-3f58179f524a is now active with session: 762d772e-a40c-4d98-a6f8-4561780bd4eb","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406476539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406489823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811,"session_id":"762d772e-a40c-4d98-a6f8-4561780bd4eb"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.406503577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} {"msg":"Activated sub_volume d91eb7d6-50ea-4fe5-8be4-3f58179f524a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:20.40651492Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 204 len 32768 data: 10 8 8 8 10 10 8 8 02/30 Read block 91 len 4096 03/30 Read block 211 len 40960 04/30 Read block 249 len 16384 05/30 Read block 69 len 20480 06/30 Write block 119 len 36864 data: 9 8 9 11 8 9 8 8 9 07/30 Write block 191 len 24576 data: 12 10 9 9 8 15 08/30 Write block 269 len 28672 data: 14 14 16 21 21 15 13 09/30 Read block 7 len 36864 10/30 Write block 457 len 4096 data: 8 11/30 Write block 210 len 16384 data: 9 9 6 8 12/30 Read block 463 len 32768 13/30 Write block 563 len 4096 data: 17 14/30 Read block 6 len 24576 15/30 Write block 453 len 40960 data: 11 6 7 8 9 7 7 8 9 11 16/30 Write block 582 len 8192 data: 13 12 17/30 Flush 18/30 Write block 198 len 4096 data: 15 19/30 Write block 111 len 32768 data: 12 11 15 14 12 8 9 8 20/30 Write block 330 len 36864 data: 6 7 5 6 8 6 6 6 8 21/30 Write block 569 len 16384 data: 11 9 8 6 22/30 Read block 485 len 12288 23/30 Write block 3 len 32768 data: 5 6 8 9 9 10 11 10 24/30 Read block 257 len 20480 25/30 Read block 523 len 4096 26/30 Write block 401 len 32768 data: 14 15 12 10 9 8 8 10 27/30 Write block 566 len 20480 data: 14 14 12 12 10 28/30 Write block 430 len 40960 data: 9 11 10 11 10 12 11 9 7 6 29/30 Write block 565 len 40960 data: 13 15 15 13 13 11 9 7 6 7 30/30 Write block 326 len 8192 data: 9 8 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:20.418557258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1811} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 212 212 212 T F T <--- 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 136 132 136 217 214 217 T F T <--- 6 120-139 136 126 136 217 205 217 F F F <--- 7 140-159 134 134 134 216 216 216 F F F 8 160-179 134 134 134 216 216 216 F F F 9 180-199 136 132 136 217 214 217 T F T <--- 10 200-219 136 134 136 217 216 217 F F F <--- 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 136 134 136 217 216 217 F F F <--- 14 280-299 128 128 128 210 210 210 F F F 15 300-319 134 134 134 216 216 216 F F F 16 320-339 126 126 126 203 203 203 T F T <--- 17 340-359 134 134 134 216 216 216 F F F 18 360-379 133 133 133 215 215 215 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 134 134 134 216 216 216 T F T <--- 21 420-439 134 134 134 216 216 216 T F T <--- 22 440-459 136 130 136 217 212 217 F F F <--- 23 460-479 136 134 136 217 216 217 F F F <--- 24 480-499 134 134 134 216 216 216 F F F 25 500-519 134 134 134 216 216 216 F F F 26 520-539 134 134 134 216 216 216 F F F 27 540-559 133 133 133 215 215 215 F F F 28 560-579 136 134 136 217 216 217 T F T <--- 29 580-599 136 127 136 217 206 217 F F F <--- Max gen: 136, Max flush: 217 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 218 218 218 F F F 1 020-039 126 126 126 205 205 205 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 137 137 137 218 218 218 F F F 6 120-139 136 136 136 217 217 217 F F F 7 140-159 134 134 134 216 216 216 F F F 8 160-179 134 134 134 216 216 216 F F F 9 180-199 137 137 137 218 218 218 F F F 10 200-219 136 136 136 217 217 217 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 134 134 134 216 216 216 F F F 16 320-339 137 137 137 218 218 218 F F F 17 340-359 134 134 134 216 216 216 F F F 18 360-379 133 133 133 215 215 215 F F F 19 380-399 131 131 131 213 213 213 F F F 20 400-419 137 137 137 218 218 218 F F F 21 420-439 137 137 137 218 218 218 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 136 136 136 217 217 217 F F F 24 480-499 134 134 134 216 216 216 F F F 25 500-519 134 134 134 216 216 216 F F F 26 520-539 134 134 134 216 216 216 F F F 27 540-559 133 133 133 215 215 215 F F F 28 560-579 137 137 137 218 218 218 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 137, Max flush: 218 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.780320086Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:34.791653173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.791713801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.802579207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.802940835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.802985098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.802996932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803014777Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"Crucible dd8e3709-0f76-4ed5-bddb-cd51dd8f4482 has session id: e793f87c-1bcb-4486-90a5-58787d3137bd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803455378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Upstairs opts: Upstairs UUID: dd8e3709-0f76-4ed5-bddb-cd51dd8f4482, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803490609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Crucible stats registered with UUID: dd8e3709-0f76-4ed5-bddb-cd51dd8f4482","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803506353Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:34.803519877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803706793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803731521Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803746235Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.80393029Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803951746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"dd8e3709-0f76-4ed5-bddb-cd51dd8f4482 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803966691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.803979654Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804006042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804020547Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804034431Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804048725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804063109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44904)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804103441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51309)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804119576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:42170)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804134541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804867299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.804914573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.80507085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805443941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[0]R flush_numbers[0..12]: [218, 205, 214, 201, 211, 218, 217, 216, 216, 218, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805472399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[0]R generation[0..12]: [137, 126, 132, 125, 129, 137, 136, 134, 134, 137, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805492745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.80550944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[1]R flush_numbers[0..12]: [218, 205, 214, 201, 211, 218, 217, 216, 216, 218, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805525465Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[1]R generation[0..12]: [137, 126, 132, 125, 129, 137, 136, 134, 134, 137, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.80554129Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805557575Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[2]R flush_numbers[0..12]: [218, 205, 214, 201, 211, 218, 217, 216, 216, 218, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805573459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[2]R generation[0..12]: [137, 126, 132, 125, 129, 137, 136, 134, 134, 137, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805589114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805605429Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805618723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805632907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805646011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805665247Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"":"downstairs","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805679991Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805692865Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"0","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.80570929Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"1","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805723164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"client":"2","":"downstairs","session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805737648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805750832Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"dd8e3709-0f76-4ed5-bddb-cd51dd8f4482 is now active with session: e793f87c-1bcb-4486-90a5-58787d3137bd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805765687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:28:34.80577823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820,"session_id":"e793f87c-1bcb-4486-90a5-58787d3137bd"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805792775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} {"msg":"Activated sub_volume dd8e3709-0f76-4ed5-bddb-cd51dd8f4482","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:34.805804248Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 32768 data: 9 8 7 7 8 6 7 9 02/30 Write block 430 len 36864 data: 10 12 11 12 11 13 12 10 8 03/30 Read block 543 len 32768 04/30 Write block 345 len 16384 data: 15 17 12 12 05/30 Write block 552 len 4096 data: 12 06/30 Read block 58 len 24576 07/30 Read block 339 len 36864 08/30 Write block 176 len 24576 data: 15 12 10 12 10 7 09/30 Read block 196 len 16384 10/30 Write block 323 len 36864 data: 10 9 10 10 10 9 8 8 9 11/30 Flush 12/30 Write block 523 len 40960 data: 7 7 7 7 11 11 10 10 14 15 13/30 Write block 117 len 16384 data: 10 9 10 9 14/30 Write block 396 len 20480 data: 9 10 11 10 11 15/30 Write block 106 len 4096 data: 12 16/30 Flush 17/30 Write block 486 len 20480 data: 9 11 9 10 11 18/30 Write block 12 len 36864 data: 12 12 10 12 12 12 9 10 7 19/30 Read block 481 len 32768 20/30 Read block 20 len 4096 21/30 Read block 153 len 12288 22/30 Read block 454 len 24576 23/30 Write block 6 len 32768 data: 10 10 11 12 11 12 13 13 24/30 Read block 16 len 36864 25/30 Write block 505 len 12288 data: 21 20 21 26/30 Read block 75 len 20480 27/30 Write block 566 len 24576 data: 16 16 14 14 12 10 28/30 Write block 358 len 12288 data: 11 9 10 29/30 Read block 327 len 8192 30/30 Write block 189 len 32768 data: 10 10 13 11 10 10 9 16 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:34.819525266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1820} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 218 218 218 F T T <--- 1 020-039 126 126 126 205 205 205 F T T <--- 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 137 138 138 218 220 220 F F F <--- 6 120-139 136 138 138 217 220 220 F F F <--- 7 140-159 134 134 134 216 216 216 F F F 8 160-179 134 138 138 216 219 219 F F F <--- 9 180-199 137 138 138 218 219 219 F T T <--- 10 200-219 136 136 136 217 217 217 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 134 134 134 216 216 216 F F F 16 320-339 137 138 138 218 219 219 T F F <--- 17 340-359 134 138 138 216 219 219 F T T <--- 18 360-379 133 133 133 215 215 215 F T T <--- 19 380-399 131 138 138 213 220 220 F F F <--- 20 400-419 137 138 138 218 220 220 F F F <--- 21 420-439 137 138 138 218 219 219 T F F <--- 22 440-459 136 136 136 217 217 217 F F F 23 460-479 136 136 136 217 217 217 F F F 24 480-499 134 134 134 216 216 216 F T T <--- 25 500-519 134 134 134 216 216 216 F T T <--- 26 520-539 134 138 138 216 220 220 F F F <--- 27 540-559 133 138 138 215 219 219 F F F <--- 28 560-579 137 137 137 218 218 218 F T T <--- 29 580-599 136 136 136 217 217 217 F F F Max gen: 138, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 139 139 139 221 221 221 F F F 2 040-059 132 132 132 214 214 214 F F F 3 060-079 125 125 125 201 201 201 F F F 4 080-099 129 129 129 211 211 211 F F F 5 100-119 138 138 138 220 220 220 F F F 6 120-139 138 138 138 220 220 220 F F F 7 140-159 134 134 134 216 216 216 F F F 8 160-179 138 138 138 219 219 219 F F F 9 180-199 139 139 139 221 221 221 F F F 10 200-219 136 136 136 217 217 217 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 128 128 128 210 210 210 F F F 15 300-319 134 134 134 216 216 216 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 139 139 139 221 221 221 F F F 19 380-399 138 138 138 220 220 220 F F F 20 400-419 138 138 138 220 220 220 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 136 136 136 217 217 217 F F F 24 480-499 139 139 139 221 221 221 F F F 25 500-519 139 139 139 221 221 221 F F F 26 520-539 138 138 138 220 220 220 F F F 27 540-559 138 138 138 219 219 219 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 139, Max flush: 221 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.233621677Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.24544405Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.245763305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.245802746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.24581552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.245834216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"Crucible 6c881584-f937-4a76-8dc5-10b03cc71689 has session id: bec4ae25-c179-45db-83f8-d2150f7bac3d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246199665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Upstairs opts: Upstairs UUID: 6c881584-f937-4a76-8dc5-10b03cc71689, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246239006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Crucible stats registered with UUID: 6c881584-f937-4a76-8dc5-10b03cc71689","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246254901Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:49.246268515Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246405426Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246430884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246445578Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246593912Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246610257Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"6c881584-f937-4a76-8dc5-10b03cc71689 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246633014Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246646318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246660632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246675697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246690101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246704305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246718719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40576)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246780128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60827)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246796362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39807)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.246811537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"io task","client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.247515017Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.247565372Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248078194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248438852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[0]R flush_numbers[0..12]: [221, 221, 214, 201, 211, 220, 220, 216, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248462729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[0]R generation[0..12]: [139, 139, 132, 125, 129, 138, 138, 134, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248478784Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248495569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[1]R flush_numbers[0..12]: [221, 221, 214, 201, 211, 220, 220, 216, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248511534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[1]R generation[0..12]: [139, 139, 132, 125, 129, 138, 138, 134, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248527768Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248543893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[2]R flush_numbers[0..12]: [221, 221, 214, 201, 211, 220, 220, 216, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248559548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[2]R generation[0..12]: [139, 139, 132, 125, 129, 138, 138, 134, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248578033Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248594548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248608002Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:28:49.248621866Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248641272Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248654706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"":"downstairs","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.24866875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248681394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"0","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248695448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"1","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248709793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"client":"2","":"downstairs","session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248723907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248737051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 6c881584-f937-4a76-8dc5-10b03cc71689 is now active with session: bec4ae25-c179-45db-83f8-d2150f7bac3d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248752105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248764969Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829,"session_id":"bec4ae25-c179-45db-83f8-d2150f7bac3d"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248778213Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} {"msg":"Activated sub_volume 6c881584-f937-4a76-8dc5-10b03cc71689","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:28:49.248789786Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 507 len 32768 data: 22 19 17 17 17 16 14 13 03/30 Write block 116 len 4096 data: 9 04/30 Write block 61 len 40960 data: 9 8 8 7 8 8 10 13 13 12 05/30 Write block 51 len 4096 data: 10 06/30 Read block 313 len 16384 07/30 Read block 459 len 36864 08/30 Flush 09/30 Write block 373 len 28672 data: 13 13 12 13 10 12 11 10/30 Read block 538 len 8192 11/30 Write block 310 len 32768 data: 11 8 6 7 7 9 11 12 12/30 Write block 132 len 40960 data: 9 9 9 12 11 13 11 10 11 10 13/30 Read block 345 len 40960 14/30 Write block 26 len 36864 data: 6 8 8 9 10 9 10 9 10 15/30 Flush 16/30 Write block 298 len 32768 data: 14 15 15 13 12 11 11 8 17/30 Write block 530 len 20480 data: 11 15 16 17 16 18/30 Read block 36 len 16384 19/30 Write block 56 len 24576 data: 11 11 10 9 9 10 20/30 Write block 89 len 16384 data: 11 9 7 9 21/30 Read block 177 len 8192 22/30 Read block 207 len 16384 23/30 Write block 254 len 12288 data: 14 16 15 24/30 Write block 510 len 4096 data: 18 25/30 Read block 527 len 12288 26/30 Read block 47 len 40960 27/30 Write block 479 len 24576 data: 12 11 11 10 9 9 28/30 Read block 545 len 12288 29/30 Read block 394 len 32768 30/30 Read block 139 len 36864 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 29 28 0 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:28:49.261774565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1829} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 139 224 224 221 F F F <--- 2 040-059 140 140 132 223 223 214 T T F <--- 3 060-079 140 140 125 223 223 201 T T F <--- 4 080-099 129 129 129 211 211 211 T T F <--- 5 100-119 140 140 138 223 223 220 F F F <--- 6 120-139 140 140 138 224 224 220 F F F <--- 7 140-159 140 140 134 224 224 216 F F F <--- 8 160-179 138 138 138 219 219 219 F F F 9 180-199 139 139 139 221 221 221 F F F 10 200-219 136 136 136 217 217 217 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 132 132 132 214 214 214 T T F <--- 13 260-279 136 136 136 217 217 217 F F F 14 280-299 128 128 128 210 210 210 T T F <--- 15 300-319 140 140 134 224 224 216 T T F <--- 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 140 140 139 224 224 221 F F F <--- 19 380-399 138 138 138 220 220 220 F F F 20 400-419 138 138 138 220 220 220 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 136 136 136 217 217 217 T T F <--- 24 480-499 139 139 139 221 221 221 T T F <--- 25 500-519 140 140 139 223 223 221 T T F <--- 26 520-539 138 138 138 220 220 220 T T F <--- 27 540-559 138 138 138 219 219 219 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 140, Max flush: 224 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 141 141 141 225 225 225 F F F 4 080-099 141 141 141 225 225 225 F F F 5 100-119 140 140 140 223 223 223 F F F 6 120-139 140 140 140 224 224 224 F F F 7 140-159 140 140 140 224 224 224 F F F 8 160-179 138 138 138 219 219 219 F F F 9 180-199 139 139 139 221 221 221 F F F 10 200-219 136 136 136 217 217 217 F F F 11 220-239 134 134 134 216 216 216 F F F 12 240-259 141 141 141 225 225 225 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 141 141 141 225 225 225 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 140 140 140 224 224 224 F F F 19 380-399 138 138 138 220 220 220 F F F 20 400-419 138 138 138 220 220 220 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 141 141 141 225 225 225 F F F 24 480-499 141 141 141 225 225 225 F F F 25 500-519 141 141 141 225 225 225 F F F 26 520-539 141 141 141 225 225 225 F F F 27 540-559 138 138 138 219 219 219 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 141, Max flush: 225 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.603468324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615196299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615517194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615555476Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.61556812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615586365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"Crucible 5d1f862c-9f9e-4e15-89de-0151debc58be has session id: acb98af0-a723-480a-8e55-6a40e302d696","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.61593984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Upstairs opts: Upstairs UUID: 5d1f862c-9f9e-4e15-89de-0151debc58be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615976071Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Crucible stats registered with UUID: 5d1f862c-9f9e-4e15-89de-0151debc58be","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.615992246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:03.616006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616136819Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616161406Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616176251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616319353Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616335688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"5d1f862c-9f9e-4e15-89de-0151debc58be active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616360306Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.61637318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616387314Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616401558Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616415422Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616429976Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616444151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:54299)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616469708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:54640)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616567307Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"ds_connection connected from Ok(127.0.0.1:49337)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.616586633Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.617283551Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.617325103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.617773917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618110367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[0]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 223, 224, 224, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618133434Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[0]R generation[0..12]: [139, 140, 141, 141, 141, 140, 140, 140, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618149719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618166054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[1]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 223, 224, 224, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618181788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[1]R generation[0..12]: [139, 140, 141, 141, 141, 140, 140, 140, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618197733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618214038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[2]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 223, 224, 224, 219, 221, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618229873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[2]R generation[0..12]: [139, 140, 141, 141, 141, 140, 140, 140, 138, 139, 136, 134]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618245597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618261732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618275176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.61828921ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618309096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.61832262Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"":"downstairs","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618336875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618351369Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"0","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618365453Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"1","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618379657Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"client":"2","":"downstairs","session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618393782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618409816Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"5d1f862c-9f9e-4e15-89de-0151debc58be is now active with session: acb98af0-a723-480a-8e55-6a40e302d696","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618424861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618437875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838,"session_id":"acb98af0-a723-480a-8e55-6a40e302d696"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618451549Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} {"msg":"Activated sub_volume 5d1f862c-9f9e-4e15-89de-0151debc58be","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:03.618463042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 24576 data: 11 11 11 12 12 14 02/30 Read block 265 len 32768 03/30 Read block 532 len 12288 04/30 Read block 549 len 36864 05/30 Read block 26 len 16384 06/30 Write block 138 len 36864 data: 12 11 12 11 11 9 8 7 7 07/30 Read block 48 len 4096 08/30 Read block 303 len 32768 09/30 Read block 284 len 28672 10/30 Read block 462 len 36864 11/30 Write block 184 len 4096 data: 7 12/30 Flush 13/30 Flush 14/30 Write block 146 len 16384 data: 8 10 12 11 15/30 Write block 478 len 4096 data: 10 16/30 Flush 17/30 Write block 499 len 40960 data: 8 12 12 10 14 18 22 21 23 20 18/30 Read block 398 len 40960 19/30 Read block 422 len 24576 20/30 Read block 496 len 8192 21/30 Read block 302 len 24576 22/30 Write block 362 len 16384 data: 9 9 8 9 23/30 Read block 281 len 24576 24/30 Read block 543 len 32768 25/30 Write block 214 len 4096 data: 8 26/30 Write block 157 len 4096 data: 10 27/30 Write block 187 len 16384 data: 10 10 11 11 28/30 Read block 194 len 28672 29/30 Read block 450 len 36864 30/30 Write block 106 len 8192 data: 13 13 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 4 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:03.630454255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1838} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 141 141 141 225 225 225 F F F 4 080-099 141 141 141 225 225 225 F F F 5 100-119 140 140 140 223 223 223 T F T <--- 6 120-139 142 140 142 226 224 226 F F F <--- 7 140-159 142 140 142 228 224 228 T F T <--- 8 160-179 138 138 138 219 219 219 F F F 9 180-199 142 139 142 226 221 226 T F T <--- 10 200-219 136 136 136 217 217 217 T F T <--- 11 220-239 142 134 142 226 216 226 F T F <--- 12 240-259 141 141 141 225 225 225 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 141 141 141 225 225 225 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 140 140 140 224 224 224 T F T <--- 19 380-399 138 138 138 220 220 220 F F F 20 400-419 138 138 138 220 220 220 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 142 141 142 228 225 228 F F F <--- 24 480-499 141 141 141 225 225 225 T F T <--- 25 500-519 141 141 141 225 225 225 T F T <--- 26 520-539 141 141 141 225 225 225 F F F 27 540-559 138 138 138 219 219 219 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 142, Max flush: 228 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 141 141 141 225 225 225 F F F 4 080-099 141 141 141 225 225 225 F F F 5 100-119 143 143 143 229 229 229 F F F 6 120-139 142 142 142 226 226 226 F F F 7 140-159 143 143 143 229 229 229 F F F 8 160-179 138 138 138 219 219 219 F F F 9 180-199 143 143 143 229 229 229 F F F 10 200-219 143 143 143 229 229 229 F F F 11 220-239 142 142 142 226 226 226 F F F 12 240-259 141 141 141 225 225 225 F F F 13 260-279 136 136 136 217 217 217 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 141 141 141 225 225 225 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 143 143 143 229 229 229 F F F 19 380-399 138 138 138 220 220 220 F F F 20 400-419 138 138 138 220 220 220 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F F F 23 460-479 142 142 142 228 228 228 F F F 24 480-499 143 143 143 229 229 229 F F F 25 500-519 143 143 143 229 229 229 F F F 26 520-539 141 141 141 225 225 225 F F F 27 540-559 138 138 138 219 219 219 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 143, Max flush: 229 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.956454859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:17.967191068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.967244964Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.977719605Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978035139Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978074891Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978086725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.97810479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"Crucible 767917e3-da00-4cb7-8191-f465c4210572 has session id: 8df17b8e-6a86-4158-a3f1-729180a58434","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978467718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Upstairs opts: Upstairs UUID: 767917e3-da00-4cb7-8191-f465c4210572, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978502859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Crucible stats registered with UUID: 767917e3-da00-4cb7-8191-f465c4210572","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978518343Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:17.978531697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978685173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978715352Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978730607Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978865647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978881592Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"767917e3-da00-4cb7-8191-f465c4210572 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978898867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978911811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978925795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978939719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.978953643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.97897671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979003728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"ds_connection connected from Ok(127.0.0.1:52549)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979024114Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"ds_connection connected from Ok(127.0.0.1:48615)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979038958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"ds_connection connected from Ok(127.0.0.1:59945)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979053833Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979800505Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979858853Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.979875698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980207357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[0]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 229, 226, 229, 219, 229, 229, 226]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980229043Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[0]R generation[0..12]: [139, 140, 141, 141, 141, 143, 142, 143, 138, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980245378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980262113Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[1]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 229, 226, 229, 219, 229, 229, 226]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980278188Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[1]R generation[0..12]: [139, 140, 141, 141, 141, 143, 142, 143, 138, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980312818Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980329493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[2]R flush_numbers[0..12]: [221, 224, 225, 225, 225, 229, 226, 229, 219, 229, 229, 226]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980348129Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[2]R generation[0..12]: [139, 140, 141, 141, 141, 143, 142, 143, 138, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980363973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980380198Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980393652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980407916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980427732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980441086Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"":"downstairs","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.98045543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980468294Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"0","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980482479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"1","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980496663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"client":"2","":"downstairs","session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980511027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:29:17.980524681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"767917e3-da00-4cb7-8191-f465c4210572 is now active with session: 8df17b8e-6a86-4158-a3f1-729180a58434","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980540746Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.98055373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847,"session_id":"8df17b8e-6a86-4158-a3f1-729180a58434"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980568764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} {"msg":"Activated sub_volume 767917e3-da00-4cb7-8191-f465c4210572","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:17.980580378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 12288 data: 12 10 10 02/30 Write block 414 len 24576 data: 12 9 11 9 8 9 03/30 Write block 259 len 16384 data: 13 13 12 13 04/30 Write block 369 len 28672 data: 6 7 9 9 14 14 13 05/30 Write block 102 len 4096 data: 16 06/30 Write block 142 len 16384 data: 12 10 9 8 07/30 Read block 44 len 36864 08/30 Read block 427 len 36864 09/30 Read block 54 len 4096 10/30 Read block 353 len 28672 11/30 Flush 12/30 Write block 384 len 24576 data: 8 14 11 13 11 10 13/30 Write block 111 len 16384 data: 13 12 16 15 14/30 Write block 225 len 40960 data: 11 13 12 12 12 13 13 15 14 15 15/30 Write block 454 len 16384 data: 7 8 9 10 16/30 Read block 483 len 28672 17/30 Write block 64 len 20480 data: 8 9 9 11 14 18/30 Read block 526 len 4096 19/30 Write block 126 len 28672 data: 9 10 11 9 6 6 10 20/30 Write block 443 len 40960 data: 8 12 13 12 7 8 8 11 13 12 21/30 Read block 282 len 16384 22/30 Read block 25 len 16384 23/30 Read block 354 len 40960 24/30 Read block 104 len 24576 25/30 Read block 23 len 24576 26/30 Read block 345 len 4096 27/30 Write block 153 len 40960 data: 10 8 8 10 11 10 10 8 7 8 28/30 Read block 453 len 24576 29/30 Read block 454 len 4096 30/30 Write block 188 len 8192 data: 11 12 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:17.992285585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1847} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 141 141 141 225 225 225 F T T <--- 4 080-099 141 141 141 225 225 225 F F F 5 100-119 143 144 144 229 230 230 F T T <--- 6 120-139 142 142 142 226 226 226 F T T <--- 7 140-159 143 144 144 229 230 230 F T T <--- 8 160-179 138 138 138 219 219 219 F T T <--- 9 180-199 143 143 143 229 229 229 F T T <--- 10 200-219 143 143 143 229 229 229 F F F 11 220-239 142 142 142 226 226 226 F T T <--- 12 240-259 141 144 144 225 230 230 F F F <--- 13 260-279 136 144 144 217 230 230 F F F <--- 14 280-299 141 141 141 225 225 225 F F F 15 300-319 141 141 141 225 225 225 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 143 144 144 229 230 230 F F F <--- 19 380-399 138 138 138 220 220 220 F T T <--- 20 400-419 138 144 144 220 230 230 F F F <--- 21 420-439 138 138 138 219 219 219 F F F 22 440-459 136 136 136 217 217 217 F T T <--- 23 460-479 142 142 142 228 228 228 F F F 24 480-499 143 143 143 229 229 229 F F F 25 500-519 143 143 143 229 229 229 F F F 26 520-539 141 141 141 225 225 225 F F F 27 540-559 138 144 144 219 230 230 T F F <--- 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 144, Max flush: 230 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 145 145 145 231 231 231 F F F 4 080-099 141 141 141 225 225 225 F F F 5 100-119 145 145 145 231 231 231 F F F 6 120-139 145 145 145 231 231 231 F F F 7 140-159 145 145 145 231 231 231 F F F 8 160-179 145 145 145 231 231 231 F F F 9 180-199 145 145 145 231 231 231 F F F 10 200-219 143 143 143 229 229 229 F F F 11 220-239 145 145 145 231 231 231 F F F 12 240-259 144 144 144 230 230 230 F F F 13 260-279 144 144 144 230 230 230 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 141 141 141 225 225 225 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 144 144 144 230 230 230 F F F 19 380-399 145 145 145 231 231 231 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 138 138 138 219 219 219 F F F 22 440-459 145 145 145 231 231 231 F F F 23 460-479 142 142 142 228 228 228 F F F 24 480-499 143 143 143 229 229 229 F F F 25 500-519 143 143 143 229 229 229 F F F 26 520-539 141 141 141 225 225 225 F F F 27 540-559 144 144 144 230 230 230 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 145, Max flush: 231 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.327772903Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.339513311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.339839408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.339882821Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.339895665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.339914361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"Crucible b8dd5479-045a-44ca-85b5-8e73072f556f has session id: 5b4ad12f-5dd3-4a86-909b-085086d8fcf4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.34024724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Upstairs opts: Upstairs UUID: b8dd5479-045a-44ca-85b5-8e73072f556f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340292253Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Crucible stats registered with UUID: b8dd5479-045a-44ca-85b5-8e73072f556f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340307968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:32.340321402Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340479289Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340503596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340521342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340674687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340689952Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"b8dd5479-045a-44ca-85b5-8e73072f556f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340703886Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.34071686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340730684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340744668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340758412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340772836Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340787201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48232)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340848569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33366)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340879418Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39402)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.340896013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.341585779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.341621769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342102713Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342441154Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[0]R flush_numbers[0..12]: [221, 224, 225, 231, 225, 231, 231, 231, 231, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.34246327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[0]R generation[0..12]: [139, 140, 141, 145, 141, 145, 145, 145, 145, 145, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342479495Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.34249606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[1]R flush_numbers[0..12]: [221, 224, 225, 231, 225, 231, 231, 231, 231, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342511845Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[1]R generation[0..12]: [139, 140, 141, 145, 141, 145, 145, 145, 145, 145, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342527559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342543554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[2]R flush_numbers[0..12]: [221, 224, 225, 231, 225, 231, 231, 231, 231, 231, 229, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342559399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[2]R generation[0..12]: [139, 140, 141, 145, 141, 145, 145, 145, 145, 145, 143, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342575083Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342592069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342605613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342619707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342639303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342652867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"":"downstairs","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342668881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342682285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"0","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.34269757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"1","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342712214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"client":"2","":"downstairs","session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342726989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342742893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"b8dd5479-045a-44ca-85b5-8e73072f556f is now active with session: 5b4ad12f-5dd3-4a86-909b-085086d8fcf4","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342758798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342771472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856,"session_id":"5b4ad12f-5dd3-4a86-909b-085086d8fcf4"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342784626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} {"msg":"Activated sub_volume b8dd5479-045a-44ca-85b5-8e73072f556f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:32.342796169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 65 len 40960 data: 10 10 12 15 14 13 10 10 10 10 02/30 Read block 191 len 32768 03/30 Read block 164 len 36864 04/30 Read block 501 len 24576 05/30 Write block 427 len 8192 data: 11 11 06/30 Read block 350 len 20480 07/30 Write block 536 len 16384 data: 14 14 15 13 08/30 Flush 09/30 Write block 214 len 16384 data: 9 8 11 10 10/30 Flush 11/30 Write block 118 len 36864 data: 10 11 10 10 12 9 10 9 10 12/30 Write block 475 len 24576 data: 9 11 10 11 13 12 13/30 Read block 530 len 4096 14/30 Read block 321 len 24576 15/30 Flush 16/30 Write block 388 len 28672 data: 12 11 9 11 10 8 11 17/30 Read block 508 len 40960 18/30 Read block 560 len 24576 19/30 Read block 464 len 32768 20/30 Read block 495 len 16384 21/30 Write block 309 len 36864 data: 11 12 9 7 8 8 10 12 13 22/30 Read block 360 len 20480 23/30 Read block 287 len 24576 24/30 Write block 163 len 28672 data: 10 10 8 8 8 8 6 25/30 Flush 26/30 Flush 27/30 Write block 500 len 8192 data: 13 13 28/30 Read block 509 len 24576 29/30 Read block 112 len 16384 30/30 Write block 158 len 20480 data: 11 11 9 8 9 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:32.35622358Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 146 145 146 232 231 232 F T F <--- 4 080-099 141 141 141 225 225 225 F F F 5 100-119 146 145 146 234 231 234 F F F <--- 6 120-139 146 145 146 234 231 234 F F F <--- 7 140-159 145 145 145 231 231 231 T F T <--- 8 160-179 146 145 146 235 231 235 T F T <--- 9 180-199 145 145 145 231 231 231 F F F 10 200-219 146 143 146 233 229 233 F F F <--- 11 220-239 145 145 145 231 231 231 F F F 12 240-259 144 144 144 230 230 230 F F F 13 260-279 144 144 144 230 230 230 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 146 141 146 235 225 235 F F F <--- 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 144 144 144 230 230 230 F F F 19 380-399 146 145 146 235 231 235 F F F <--- 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 138 146 232 219 232 F F F <--- 22 440-459 145 145 145 231 231 231 F F F 23 460-479 146 142 146 234 228 234 F F F <--- 24 480-499 146 143 146 234 229 234 F F F <--- 25 500-519 143 143 143 229 229 229 T F T <--- 26 520-539 146 141 146 232 225 232 F F F <--- 27 540-559 144 144 144 230 230 230 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 146, Max flush: 235 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 140 140 140 224 224 224 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 146 146 146 232 232 232 F F F 4 080-099 141 141 141 225 225 225 F F F 5 100-119 146 146 146 234 234 234 F F F 6 120-139 146 146 146 234 234 234 F F F 7 140-159 147 147 147 236 236 236 F F F 8 160-179 147 147 147 236 236 236 F F F 9 180-199 145 145 145 231 231 231 F F F 10 200-219 146 146 146 233 233 233 F F F 11 220-239 145 145 145 231 231 231 F F F 12 240-259 144 144 144 230 230 230 F F F 13 260-279 144 144 144 230 230 230 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 146 146 146 235 235 235 F F F 16 320-339 138 138 138 219 219 219 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 144 144 144 230 230 230 F F F 19 380-399 146 146 146 235 235 235 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 146 146 232 232 232 F F F 22 440-459 145 145 145 231 231 231 F F F 23 460-479 146 146 146 234 234 234 F F F 24 480-499 146 146 146 234 234 234 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 144 144 144 230 230 230 F F F 28 560-579 139 139 139 221 221 221 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 147, Max flush: 236 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.687934053Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.699743632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700061227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700099488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700111552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700129797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"Crucible 513917df-86f6-4267-9e47-41df57d4d610 has session id: 6bcb11f4-fb00-4103-9422-3c7886a67e69","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700478051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Upstairs opts: Upstairs UUID: 513917df-86f6-4267-9e47-41df57d4d610, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700513022Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Crucible stats registered with UUID: 513917df-86f6-4267-9e47-41df57d4d610","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700528176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:46.70054176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700713981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700750082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700765847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700903478Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700918602Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"513917df-86f6-4267-9e47-41df57d4d610 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700942039Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700954993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700969287Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700983732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.700997686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.70101243Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701029805Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"ds_connection connected from Ok(127.0.0.1:34935)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701200096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"ds_connection connected from Ok(127.0.0.1:38025)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701228095Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"ds_connection connected from Ok(127.0.0.1:53876)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701244379Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701806847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.701843208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702358261Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702693691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[0]R flush_numbers[0..12]: [221, 224, 225, 232, 225, 234, 234, 236, 236, 231, 233, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702716628Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[0]R generation[0..12]: [139, 140, 141, 146, 141, 146, 146, 147, 147, 145, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702733533Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702749888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[1]R flush_numbers[0..12]: [221, 224, 225, 232, 225, 234, 234, 236, 236, 231, 233, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702766023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[1]R generation[0..12]: [139, 140, 141, 146, 141, 146, 146, 147, 147, 145, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702781788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702798172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[2]R flush_numbers[0..12]: [221, 224, 225, 232, 225, 234, 234, 236, 236, 231, 233, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702817268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[2]R generation[0..12]: [139, 140, 141, 146, 141, 146, 146, 147, 147, 145, 146, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702832993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702849118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702862572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Generation requested: 148 >= found:148","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702882948Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702896902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702910706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"":"downstairs","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.70292506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702937804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"0","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702951798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"1","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702965782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"client":"2","":"downstairs","session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.702980267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.703008375Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"513917df-86f6-4267-9e47-41df57d4d610 is now active with session: 6bcb11f4-fb00-4103-9422-3c7886a67e69","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.70302372Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.703036693Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865,"session_id":"6bcb11f4-fb00-4103-9422-3c7886a67e69"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.703052088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} {"msg":"Activated sub_volume 513917df-86f6-4267-9e47-41df57d4d610","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:29:46.703063742Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 36864 data: 12 12 12 11 11 10 11 11 11 02/30 Write block 90 len 36864 data: 10 8 10 10 8 9 9 12 11 03/30 Write block 29 len 8192 data: 10 11 04/30 Write block 60 len 36864 data: 10 11 9 9 9 11 11 13 16 05/30 Write block 565 len 4096 data: 14 06/30 Read block 470 len 8192 07/30 Read block 35 len 32768 08/30 Write block 249 len 16384 data: 16 16 11 10 09/30 Flush 10/30 Write block 489 len 8192 data: 11 12 11/30 Read block 316 len 8192 12/30 Write block 442 len 40960 data: 6 9 13 14 13 8 9 9 12 14 13/30 Read block 7 len 4096 14/30 Flush 15/30 Flush 16/30 Write block 453 len 4096 data: 12 17/30 Write block 542 len 28672 data: 12 11 13 11 11 10 11 18/30 Read block 545 len 20480 19/30 Read block 496 len 12288 20/30 Read block 470 len 32768 21/30 Write block 135 len 28672 data: 13 12 14 13 12 13 12 22/30 Read block 436 len 40960 23/30 Read block 258 len 16384 24/30 Write block 195 len 28672 data: 10 17 17 16 14 13 11 25/30 Read block 128 len 16384 26/30 Write block 368 len 36864 data: 5 7 8 10 10 15 15 14 14 27/30 Read block 342 len 12288 28/30 Read block 210 len 32768 29/30 Read block 117 len 24576 30/30 Read block 512 len 4096 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 2 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 25 25 Done 30 30 5 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:29:46.715696165Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1865} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 148 148 140 237 237 224 F F T <--- 2 040-059 141 141 141 225 225 225 F F F 3 060-079 148 148 146 237 237 232 F F T <--- 4 080-099 148 148 141 237 237 225 F F T <--- 5 100-119 146 146 146 234 234 234 F F F 6 120-139 146 146 146 234 234 234 T T F <--- 7 140-159 147 147 147 236 236 236 T T F <--- 8 160-179 147 147 147 236 236 236 F F F 9 180-199 145 145 145 231 231 231 T T F <--- 10 200-219 146 146 146 233 233 233 T T F <--- 11 220-239 145 145 145 231 231 231 F F F 12 240-259 148 148 144 237 237 230 F F F <--- 13 260-279 144 144 144 230 230 230 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 148 148 146 237 237 235 F F T <--- 16 320-339 148 148 138 237 237 219 F F T <--- 17 340-359 139 139 139 221 221 221 F F F 18 360-379 144 144 144 230 230 230 T T F <--- 19 380-399 146 146 146 235 235 235 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 146 146 232 232 232 F F F 22 440-459 148 148 145 238 238 231 T T F <--- 23 460-479 146 146 146 234 234 234 F F F 24 480-499 148 148 146 238 238 234 F F F <--- 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 144 144 144 230 230 230 T T F <--- 28 560-579 148 148 139 237 237 221 F F T <--- 29 580-599 136 136 136 217 217 217 F F F Max gen: 148, Max flush: 238 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 F F F 1 020-039 148 148 148 237 237 237 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 148 148 148 237 237 237 F F F 4 080-099 148 148 148 237 237 237 F F F 5 100-119 146 146 146 234 234 234 F F F 6 120-139 149 149 149 239 239 239 F F F 7 140-159 149 149 149 239 239 239 F F F 8 160-179 147 147 147 236 236 236 F F F 9 180-199 149 149 149 239 239 239 F F F 10 200-219 149 149 149 239 239 239 F F F 11 220-239 145 145 145 231 231 231 F F F 12 240-259 148 148 148 237 237 237 F F F 13 260-279 144 144 144 230 230 230 F F F 14 280-299 141 141 141 225 225 225 F F F 15 300-319 148 148 148 237 237 237 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 139 139 139 221 221 221 F F F 18 360-379 149 149 149 239 239 239 F F F 19 380-399 146 146 146 235 235 235 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 146 146 232 232 232 F F F 22 440-459 149 149 149 239 239 239 F F F 23 460-479 146 146 146 234 234 234 F F F 24 480-499 148 148 148 238 238 238 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 136 136 136 217 217 217 F F F Max gen: 149, Max flush: 239 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.054566712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.066281933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.066587954Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.066624975Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.066637069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.066654974Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"Crucible 1c326f78-2b83-4392-ab9a-5dba5327d487 has session id: 26bfd8f0-7372-47da-99aa-5876f7bfa96f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067002728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Upstairs opts: Upstairs UUID: 1c326f78-2b83-4392-ab9a-5dba5327d487, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067060045Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Crucible stats registered with UUID: 1c326f78-2b83-4392-ab9a-5dba5327d487","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06707756Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:01.067091614Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067249611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067285892Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067304838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067461394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067475559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"1c326f78-2b83-4392-ab9a-5dba5327d487 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067506318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067520632Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067534766Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06754852Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067599766Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067625003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067640078Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39084)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067655612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64736)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067670737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63582)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.067685921Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.068462063Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.068494292Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.068750329Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06905718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[0]R flush_numbers[0..12]: [221, 237, 225, 237, 237, 234, 239, 239, 236, 239, 239, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069077876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[0]R generation[0..12]: [139, 148, 141, 148, 148, 146, 149, 149, 147, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069094391Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069110596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[1]R flush_numbers[0..12]: [221, 237, 225, 237, 237, 234, 239, 239, 236, 239, 239, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069126321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[1]R generation[0..12]: [139, 148, 141, 148, 148, 146, 149, 149, 147, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069142655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06915941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[2]R flush_numbers[0..12]: [221, 237, 225, 237, 237, 234, 239, 239, 236, 239, 239, 231]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069175225Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[2]R generation[0..12]: [139, 148, 141, 148, 148, 146, 149, 149, 147, 149, 149, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06919088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069207285Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069220829Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069241335Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069254839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069268303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"":"downstairs","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069284528Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069297382Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"0","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069311126Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"1","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.06932475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"client":"2","":"downstairs","session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069339014Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069356079Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"1c326f78-2b83-4392-ab9a-5dba5327d487 is now active with session: 26bfd8f0-7372-47da-99aa-5876f7bfa96f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069371043Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069383977Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874,"session_id":"26bfd8f0-7372-47da-99aa-5876f7bfa96f"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069397511Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} {"msg":"Activated sub_volume 1c326f78-2b83-4392-ab9a-5dba5327d487","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:01.069408995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 368 len 40960 data: 6 8 9 11 11 16 16 15 15 11 02/30 Write block 449 len 12288 data: 10 13 15 03/30 Write block 489 len 28672 data: 12 13 12 13 11 12 10 04/30 Write block 582 len 28672 data: 14 13 10 10 11 9 8 05/30 Read block 497 len 40960 06/30 Read block 463 len 20480 07/30 Read block 414 len 36864 08/30 Read block 498 len 12288 09/30 Write block 165 len 8192 data: 9 9 10/30 Read block 528 len 4096 11/30 Read block 238 len 24576 12/30 Write block 382 len 20480 data: 11 8 9 15 12 13/30 Write block 228 len 12288 data: 13 13 14 14/30 Write block 398 len 8192 data: 12 11 15/30 Read block 145 len 16384 16/30 Read block 573 len 4096 17/30 Write block 128 len 28672 data: 12 10 7 7 11 10 10 18/30 Read block 196 len 12288 19/30 Write block 67 len 16384 data: 14 17 15 14 20/30 Write block 25 len 36864 data: 6 7 9 9 11 12 10 11 10 21/30 Write block 215 len 8192 data: 9 12 22/30 Write block 345 len 32768 data: 16 18 13 13 13 13 11 14 23/30 Read block 8 len 12288 24/30 Read block 459 len 4096 25/30 Write block 279 len 8192 data: 10 12 26/30 Write block 13 len 16384 data: 14 11 13 13 27/30 Write block 473 len 4096 data: 9 28/30 Read block 360 len 36864 29/30 Read block 350 len 28672 30/30 Write block 203 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 2 29 Done 29 4 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:01.079673773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1874} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 221 221 221 T F T <--- 1 020-039 148 148 148 237 237 237 T F T <--- 2 040-059 141 141 141 225 225 225 F F F 3 060-079 148 148 148 237 237 237 T F T <--- 4 080-099 148 148 148 237 237 237 F F F 5 100-119 146 146 146 234 234 234 F F F 6 120-139 149 149 149 239 239 239 T F T <--- 7 140-159 149 149 149 239 239 239 F F F 8 160-179 147 147 147 236 236 236 T F T <--- 9 180-199 149 149 149 239 239 239 F F F 10 200-219 149 149 149 239 239 239 T F T <--- 11 220-239 145 145 145 231 231 231 T F T <--- 12 240-259 148 148 148 237 237 237 F F F 13 260-279 144 144 144 230 230 230 T F T <--- 14 280-299 141 141 141 225 225 225 T F T <--- 15 300-319 148 148 148 237 237 237 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 139 139 139 221 221 221 T F T <--- 18 360-379 149 149 149 239 239 239 T T T 19 380-399 146 146 146 235 235 235 T F T <--- 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 146 146 232 232 232 F F F 22 440-459 149 149 149 239 239 239 T T T 23 460-479 146 146 146 234 234 234 T F T <--- 24 480-499 148 148 148 238 238 238 T T T 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 136 136 136 217 217 217 T T T Max gen: 149, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 240 240 240 F F F 1 020-039 150 150 150 240 240 240 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 150 150 150 240 240 240 F F F 4 080-099 148 148 148 237 237 237 F F F 5 100-119 146 146 146 234 234 234 F F F 6 120-139 150 150 150 240 240 240 F F F 7 140-159 149 149 149 239 239 239 F F F 8 160-179 150 150 150 240 240 240 F F F 9 180-199 149 149 149 239 239 239 F F F 10 200-219 150 150 150 240 240 240 F F F 11 220-239 150 150 150 240 240 240 F F F 12 240-259 148 148 148 237 237 237 F F F 13 260-279 150 150 150 240 240 240 F F F 14 280-299 150 150 150 240 240 240 F F F 15 300-319 148 148 148 237 237 237 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 150 150 150 240 240 240 F F F 18 360-379 150 150 150 240 240 240 F F F 19 380-399 150 150 150 240 240 240 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 146 146 232 232 232 F F F 22 440-459 150 150 150 240 240 240 F F F 23 460-479 150 150 150 240 240 240 F F F 24 480-499 150 150 150 240 240 240 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 150 150 150 240 240 240 F F F Max gen: 150, Max flush: 240 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.426371543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:15.437133479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.437185035Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.447625485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.447914061Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.447952493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.447968217Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.447986143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"Crucible 2c12c3e8-6a59-40b7-9b92-32da44efd6d1 has session id: ba09a6f1-facb-4dde-bdc5-621c800546c6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.44834595Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Upstairs opts: Upstairs UUID: 2c12c3e8-6a59-40b7-9b92-32da44efd6d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.44837793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Crucible stats registered with UUID: 2c12c3e8-6a59-40b7-9b92-32da44efd6d1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448393084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:15.448406588Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448565605Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448589343Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448604177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448760454Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448777389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"2c12c3e8-6a59-40b7-9b92-32da44efd6d1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448793683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448806597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448820481Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448834676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.44885049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448864795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448879279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57982)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448938337Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41758)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448955132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48663)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.448970136Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.449728192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.449775336Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.449792091Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450163862Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 225, 240, 237, 234, 240, 239, 240, 239, 240, 240]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450187669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[0]R generation[0..12]: [150, 150, 141, 150, 148, 146, 150, 149, 150, 149, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450203834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450220439Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 225, 240, 237, 234, 240, 239, 240, 239, 240, 240]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450236103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[1]R generation[0..12]: [150, 150, 141, 150, 148, 146, 150, 149, 150, 149, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450254979Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450272064Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 225, 240, 237, 234, 240, 239, 240, 239, 240, 240]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450288249Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[2]R generation[0..12]: [150, 150, 141, 150, 148, 146, 150, 149, 150, 149, 150, 150]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450303843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450320488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Max found gen is 151","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450334212Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 152 >= found:151","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450353288Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450366662Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450380366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"":"downstairs","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.45039463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450407444Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"0","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450421729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"1","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450435453Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"client":"2","":"downstairs","session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450449627Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","vDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450466112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"2c12c3e8-6a59-40b7-9b92-32da44efd6d1 is now active with session: ba09a6f1-facb-4dde-bdc5-621c800546c6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450483437Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450496201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883,"session_id":"ba09a6f1-facb-4dde-bdc5-621c800546c6"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450509675Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} {"msg":"Activated sub_volume 2c12c3e8-6a59-40b7-9b92-32da44efd6d1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:15.450522158Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 93 len 36864 data: 11 9 10 10 13 12 11 14 13 02/30 Read block 79 len 8192 03/30 Read block 268 len 28672 04/30 Write block 427 len 8192 data: 12 12 05/30 Write block 198 len 4096 data: 17 06/30 Read block 420 len 20480 07/30 Read block 310 len 12288 08/30 Flush 09/30 Write block 120 len 24576 data: 11 11 13 10 11 10 10/30 Read block 280 len 40960 11/30 Flush 12/30 Write block 299 len 32768 data: 16 16 14 13 12 12 9 8 13/30 Write block 109 len 32768 data: 11 13 14 13 17 16 13 10 14/30 Read block 533 len 4096 15/30 Write block 248 len 28672 data: 15 17 17 12 11 10 15 16/30 Write block 365 len 28672 data: 10 6 5 7 9 10 12 17/30 Write block 18 len 16384 data: 10 11 8 8 18/30 Write block 149 len 20480 data: 12 10 8 9 11 19/30 Read block 343 len 20480 20/30 Write block 90 len 32768 data: 11 9 11 12 10 11 11 14 21/30 Read block 232 len 12288 22/30 Flush 23/30 Write block 232 len 8192 data: 16 15 24/30 Read block 78 len 20480 25/30 Read block 292 len 28672 26/30 Read block 185 len 36864 27/30 Read block 237 len 20480 28/30 Read block 301 len 32768 29/30 Write block 582 len 20480 data: 15 14 11 11 12 30/30 Write block 105 len 36864 data: 14 14 14 9 12 14 15 14 18 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 2 31 Done 2 29 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:15.46354828Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1883} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 152 152 240 243 243 F F F <--- 1 020-039 150 152 152 240 243 243 F F F <--- 2 040-059 141 141 141 225 225 225 F F F 3 060-079 150 150 150 240 240 240 F F F 4 080-099 148 152 152 237 243 243 T F F <--- 5 100-119 146 152 152 234 243 243 T T T <--- 6 120-139 150 152 152 240 242 242 F F F <--- 7 140-159 149 152 152 239 243 243 F F F <--- 8 160-179 150 150 150 240 240 240 F F F 9 180-199 149 152 152 239 241 241 F F F <--- 10 200-219 150 150 150 240 240 240 F F F 11 220-239 150 150 150 240 240 240 F T T <--- 12 240-259 148 152 152 237 243 243 F F F <--- 13 260-279 150 150 150 240 240 240 F F F 14 280-299 150 152 152 240 243 243 F F F <--- 15 300-319 148 152 152 237 243 243 F F F <--- 16 320-339 148 148 148 237 237 237 F F F 17 340-359 150 150 150 240 240 240 F F F 18 360-379 150 152 152 240 243 243 F F F <--- 19 380-399 150 150 150 240 240 240 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 146 152 152 232 241 241 F F F <--- 22 440-459 150 150 150 240 240 240 F F F 23 460-479 150 150 150 240 240 240 F F F 24 480-499 150 150 150 240 240 240 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 150 150 150 240 240 240 F T T <--- Max gen: 152, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F F F 1 020-039 152 152 152 243 243 243 F F F 2 040-059 141 141 141 225 225 225 F F F 3 060-079 150 150 150 240 240 240 F F F 4 080-099 152 152 152 243 243 243 F F F 5 100-119 153 153 153 244 244 244 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F F F 8 160-179 150 150 150 240 240 240 F F F 9 180-199 152 152 152 241 241 241 F F F 10 200-219 150 150 150 240 240 240 F F F 11 220-239 153 153 153 244 244 244 F F F 12 240-259 152 152 152 243 243 243 F F F 13 260-279 150 150 150 240 240 240 F F F 14 280-299 152 152 152 243 243 243 F F F 15 300-319 152 152 152 243 243 243 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 150 150 150 240 240 240 F F F 18 360-379 152 152 152 243 243 243 F F F 19 380-399 150 150 150 240 240 240 F F F 20 400-419 144 144 144 230 230 230 F F F 21 420-439 152 152 152 241 241 241 F F F 22 440-459 150 150 150 240 240 240 F F F 23 460-479 150 150 150 240 240 240 F F F 24 480-499 150 150 150 240 240 240 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 153, Max flush: 244 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.795976027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.807717205Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.80803337Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808071121Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808083535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.80810138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"Crucible a4191123-cb6e-44dd-97ab-5c7af1c3b274 has session id: e4dc2443-24c6-4694-a403-039a51c3cb58","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808483364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Upstairs opts: Upstairs UUID: a4191123-cb6e-44dd-97ab-5c7af1c3b274, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808518494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Crucible stats registered with UUID: a4191123-cb6e-44dd-97ab-5c7af1c3b274","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808534539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:29.808547933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808712012Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808746292Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808761947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808903319Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808920524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"a4191123-cb6e-44dd-97ab-5c7af1c3b274 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808935759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808948772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808962887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808976881Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.808990545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809004989Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809019343Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:47604)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809112641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:55504)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809226675Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"ds_connection connected from Ok(127.0.0.1:48077)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809244381Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.809937797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810072767Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810197775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810520001Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[0]R flush_numbers[0..12]: [243, 243, 225, 240, 243, 244, 242, 243, 240, 241, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810542847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[0]R generation[0..12]: [152, 152, 141, 150, 152, 153, 152, 152, 150, 152, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810559122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810575907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[1]R flush_numbers[0..12]: [243, 243, 225, 240, 243, 244, 242, 243, 240, 241, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810591692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[1]R generation[0..12]: [152, 152, 141, 150, 152, 153, 152, 152, 150, 152, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810607387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810623732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[2]R flush_numbers[0..12]: [243, 243, 225, 240, 243, 244, 242, 243, 240, 241, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810639906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[2]R generation[0..12]: [152, 152, 141, 150, 152, 153, 152, 152, 150, 152, 150, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810655341Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810671716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.81068491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810699184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Next flush: 245","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-17T14:30:29.810721041Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810734985Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"":"downstairs","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810749509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810762473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"0","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810776827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"1","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810791101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"client":"2","":"downstairs","session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810805806Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.81081933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"a4191123-cb6e-44dd-97ab-5c7af1c3b274 is now active with session: e4dc2443-24c6-4694-a403-039a51c3cb58","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:30:29.810834744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810850149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892,"session_id":"e4dc2443-24c6-4694-a403-039a51c3cb58"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810863833Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} {"msg":"Activated sub_volume a4191123-cb6e-44dd-97ab-5c7af1c3b274","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:29.810875407Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 285 len 16384 data: 13 12 13 15 02/30 Read block 339 len 40960 03/30 Read block 271 len 12288 04/30 Write block 364 len 8192 data: 9 11 05/30 Write block 189 len 28672 data: 13 12 14 12 11 11 11 06/30 Read block 128 len 36864 07/30 Write block 60 len 20480 data: 11 12 10 10 10 08/30 Write block 399 len 16384 data: 12 12 15 16 09/30 Write block 160 len 4096 data: 10 10/30 Write block 456 len 36864 data: 10 11 8 8 9 10 12 10 9 11/30 Read block 70 len 40960 12/30 Read block 200 len 40960 13/30 Write block 456 len 28672 data: 11 12 9 9 10 11 13 14/30 Read block 188 len 4096 15/30 Read block 126 len 28672 16/30 Flush 17/30 Write block 466 len 28672 data: 13 14 12 10 7 8 10 18/30 Write block 252 len 36864 data: 12 11 16 17 16 15 13 14 14 19/30 Write block 263 len 20480 data: 12 10 8 12 16 20/30 Write block 194 len 20480 data: 12 12 18 18 18 21/30 Read block 89 len 8192 22/30 Read block 455 len 24576 23/30 Write block 26 len 36864 data: 8 10 10 12 13 11 12 11 11 24/30 Write block 222 len 20480 data: 12 11 11 12 14 25/30 Read block 170 len 36864 26/30 Write block 55 len 4096 data: 11 27/30 Write block 293 len 32768 data: 11 14 12 14 12 15 17 17 28/30 Read block 77 len 28672 29/30 Read block 418 len 8192 30/30 Write block 545 len 28672 data: 12 12 11 12 11 12 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:29.823638359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1892} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F F F 1 020-039 152 152 152 243 243 243 T F T <--- 2 040-059 141 141 141 225 225 225 T F T <--- 3 060-079 154 150 154 245 240 245 F F F <--- 4 080-099 152 152 152 243 243 243 F F F 5 100-119 153 153 153 244 244 244 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F F F 8 160-179 154 150 154 245 240 245 F F F <--- 9 180-199 154 152 154 245 241 245 T F T <--- 10 200-219 150 150 150 240 240 240 F F F 11 220-239 153 153 153 244 244 244 T F T <--- 12 240-259 152 152 152 243 243 243 T F T <--- 13 260-279 150 150 150 240 240 240 T F T <--- 14 280-299 154 152 154 245 243 245 T T T <--- 15 300-319 152 152 152 243 243 243 T F T <--- 16 320-339 148 148 148 237 237 237 F F F 17 340-359 150 150 150 240 240 240 F F F 18 360-379 154 152 154 245 243 245 F F F <--- 19 380-399 154 150 154 245 240 245 F F F <--- 20 400-419 154 144 154 245 230 245 F F F <--- 21 420-439 152 152 152 241 241 241 F F F 22 440-459 154 150 154 245 240 245 F F F <--- 23 460-479 154 150 154 245 240 245 T F T <--- 24 480-499 150 150 150 240 240 240 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 149 149 149 239 239 239 T F T <--- 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 154, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F F F 1 020-039 155 155 155 246 246 246 F F F 2 040-059 155 155 155 246 246 246 F F F 3 060-079 154 154 154 245 245 245 F F F 4 080-099 152 152 152 243 243 243 F F F 5 100-119 153 153 153 244 244 244 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F F F 8 160-179 154 154 154 245 245 245 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 150 150 150 240 240 240 F F F 11 220-239 155 155 155 246 246 246 F F F 12 240-259 155 155 155 246 246 246 F F F 13 260-279 155 155 155 246 246 246 F F F 14 280-299 155 155 155 246 246 246 F F F 15 300-319 155 155 155 246 246 246 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 150 150 150 240 240 240 F F F 18 360-379 154 154 154 245 245 245 F F F 19 380-399 154 154 154 245 245 245 F F F 20 400-419 154 154 154 245 245 245 F F F 21 420-439 152 152 152 241 241 241 F F F 22 440-459 154 154 154 245 245 245 F F F 23 460-479 155 155 155 246 246 246 F F F 24 480-499 150 150 150 240 240 240 F F F 25 500-519 147 147 147 236 236 236 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 155, Max flush: 246 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.166554615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.178316149Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.178630273Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.178673175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.1786896Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.178707666Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"Crucible d8cb25a1-2ba5-4077-9aeb-1334762d3b26 has session id: 144f259b-9e3e-4429-9253-af1a65032e17","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179031082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Upstairs opts: Upstairs UUID: d8cb25a1-2ba5-4077-9aeb-1334762d3b26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179065952Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Crucible stats registered with UUID: d8cb25a1-2ba5-4077-9aeb-1334762d3b26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179082087Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:44.179095691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.17925792Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.17929193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179307364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179451377Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179472443Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"d8cb25a1-2ba5-4077-9aeb-1334762d3b26 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179484817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179497691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179512585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179529711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179543545Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179581966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179604953Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"ds_connection connected from Ok(127.0.0.1:53812)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179637813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"ds_connection connected from Ok(127.0.0.1:57783)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179653007Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"ds_connection connected from Ok(127.0.0.1:36396)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.179668002Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.180418825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.180458787Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.180914493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181252734Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[0]R flush_numbers[0..12]: [243, 246, 246, 245, 243, 244, 242, 243, 245, 246, 240, 246]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181276301Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[0]R generation[0..12]: [152, 155, 155, 154, 152, 153, 152, 152, 154, 155, 150, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181292256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181308581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[1]R flush_numbers[0..12]: [243, 246, 246, 245, 243, 244, 242, 243, 245, 246, 240, 246]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181324635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[1]R generation[0..12]: [152, 155, 155, 154, 152, 153, 152, 152, 154, 155, 150, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181343711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181360576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[2]R flush_numbers[0..12]: [243, 246, 246, 245, 243, 244, 242, 243, 245, 246, 240, 246]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181376721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[2]R generation[0..12]: [152, 155, 155, 154, 152, 153, 152, 152, 154, 155, 150, 155]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181392566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.18140879Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181422635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:30:44.181436729Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181454404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181467738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"":"downstairs","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181481472Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181494186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"0","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.18150825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"1","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181522264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"client":"2","":"downstairs","session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181536569Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181555194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"d8cb25a1-2ba5-4077-9aeb-1334762d3b26 is now active with session: 144f259b-9e3e-4429-9253-af1a65032e17","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181570179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181583142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901,"session_id":"144f259b-9e3e-4429-9253-af1a65032e17"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.181596686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} {"msg":"Activated sub_volume d8cb25a1-2ba5-4077-9aeb-1334762d3b26","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:44.18160812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 20480 data: 13 12 17 17 16 02/30 Flush 03/30 Write block 517 len 8192 data: 14 12 04/30 Write block 410 len 4096 data: 7 05/30 Read block 211 len 36864 06/30 Write block 41 len 8192 data: 9 11 07/30 Read block 407 len 4096 08/30 Write block 110 len 16384 data: 15 16 15 19 09/30 Read block 572 len 12288 10/30 Read block 384 len 32768 11/30 Write block 204 len 36864 data: 11 9 9 9 11 11 10 10 7 12/30 Write block 345 len 4096 data: 17 13/30 Write block 75 len 24576 data: 10 12 10 11 10 8 14/30 Read block 456 len 36864 15/30 Read block 133 len 12288 16/30 Write block 392 len 40960 data: 11 9 12 8 10 11 13 13 13 16 17/30 Read block 162 len 12288 18/30 Flush 19/30 Write block 363 len 16384 data: 10 10 12 7 20/30 Write block 222 len 8192 data: 13 12 21/30 Write block 443 len 32768 data: 10 14 15 14 9 10 11 14 22/30 Read block 425 len 16384 23/30 Read block 112 len 12288 24/30 Read block 374 len 4096 25/30 Read block 460 len 28672 26/30 Write block 165 len 32768 data: 10 10 9 9 7 5 7 8 27/30 Write block 480 len 28672 data: 13 12 11 10 10 10 10 28/30 Write block 77 len 40960 data: 11 12 11 9 9 9 10 10 9 10 29/30 Write block 433 len 28672 data: 13 12 14 13 11 9 7 30/30 Read block 97 len 4096 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 23 0 23 Done 28 5 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1001)) Some(JobId(1017)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:44.193140526Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1901} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F F F 1 020-039 155 155 155 246 246 246 F F F 2 040-059 156 155 156 248 246 248 F T F <--- 3 060-079 156 154 156 248 245 248 T F T <--- 4 080-099 156 152 156 248 243 248 T F T <--- 5 100-119 156 153 156 248 244 248 F F F <--- 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F F F 8 160-179 154 154 154 245 245 245 T F T <--- 9 180-199 155 155 155 246 246 246 F F F 10 200-219 156 150 156 248 240 248 F F F <--- 11 220-239 155 155 155 246 246 246 T F T <--- 12 240-259 155 155 155 246 246 246 F F F 13 260-279 155 155 155 246 246 246 F F F 14 280-299 155 155 155 246 246 246 F F F 15 300-319 155 155 155 246 246 246 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 156 150 156 248 240 248 F F F <--- 18 360-379 156 156 156 247 247 247 T F T <--- 19 380-399 156 154 156 248 245 248 F F F <--- 20 400-419 156 154 156 248 245 248 F T F <--- 21 420-439 152 152 152 241 241 241 T F T <--- 22 440-459 154 154 154 245 245 245 T F T <--- 23 460-479 155 155 155 246 246 246 F F F 24 480-499 150 150 150 240 240 240 T F T <--- 25 500-519 156 147 156 248 236 248 F T F <--- 26 520-539 146 146 146 232 232 232 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 156, Max flush: 248 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F F F 1 020-039 155 155 155 246 246 246 F F F 2 040-059 156 156 156 248 248 248 F F F 3 060-079 157 157 157 249 249 249 F F F 4 080-099 157 157 157 249 249 249 F F F 5 100-119 156 156 156 248 248 248 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F F F 8 160-179 157 157 157 249 249 249 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 156 156 156 248 248 248 F F F 11 220-239 157 157 157 249 249 249 F F F 12 240-259 155 155 155 246 246 246 F F F 13 260-279 155 155 155 246 246 246 F F F 14 280-299 155 155 155 246 246 246 F F F 15 300-319 155 155 155 246 246 246 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 156 156 156 248 248 248 F F F 18 360-379 157 157 157 249 249 249 F F F 19 380-399 156 156 156 248 248 248 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 157 157 157 249 249 249 F F F 22 440-459 157 157 157 249 249 249 F F F 23 460-479 155 155 155 246 246 246 F F F 24 480-499 157 157 157 249 249 249 F F F 25 500-519 156 156 156 248 248 248 F F F 26 520-539 146 146 146 232 232 232 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 157, Max flush: 249 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.530298452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:58.541022197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.541076643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.551496048Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.551785814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.551823935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.551836719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.551854725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"Crucible 6eabf0dc-d40d-41d4-8ba6-d868e93a0365 has session id: 61302a6e-f3ea-44c4-8b01-ead2622693c2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.55220731Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Upstairs opts: Upstairs UUID: 6eabf0dc-d40d-41d4-8ba6-d868e93a0365, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.55224371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Crucible stats registered with UUID: 6eabf0dc-d40d-41d4-8ba6-d868e93a0365","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552262816Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:58.55227649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552430966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552455794Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552470548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552626455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.55264365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"6eabf0dc-d40d-41d4-8ba6-d868e93a0365 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552665206Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.55267837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552692554Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552706959Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552721533Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552758184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552784162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39465)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552799966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65462)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552815481Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42958)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.552902347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.553607027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.553656762Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.55368508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554038745Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[0]R flush_numbers[0..12]: [243, 246, 248, 249, 249, 248, 242, 243, 249, 246, 248, 249]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554064023Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[0]R generation[0..12]: [152, 155, 156, 157, 157, 156, 152, 152, 157, 155, 156, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554081088Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554097843Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[1]R flush_numbers[0..12]: [243, 246, 248, 249, 249, 248, 242, 243, 249, 246, 248, 249]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554113978Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[1]R generation[0..12]: [152, 155, 156, 157, 157, 156, 152, 152, 157, 155, 156, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554129932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554146257Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[2]R flush_numbers[0..12]: [243, 246, 248, 249, 249, 248, 242, 243, 249, 246, 248, 249]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554162452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[2]R generation[0..12]: [152, 155, 156, 157, 157, 156, 152, 152, 157, 155, 156, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554177937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554194132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-17T14:30:58.554211207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554232093Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554245597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554259291Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"":"downstairs","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554273295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554286029Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"0","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554300233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"client":"1","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554314888Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 client":"2","":"downstairs","session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554332263Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554345997Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"6eabf0dc-d40d-41d4-8ba6-d868e93a0365 is now active with session: 61302a6e-f3ea-44c4-8b01-ead2622693c2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554361011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554373945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910,"session_id":"61302a6e-f3ea-44c4-8b01-ead2622693c2"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554387509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} {"msg":"Activated sub_volume 6eabf0dc-d40d-41d4-8ba6-d868e93a0365","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:30:58.554399193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 48 len 16384 data: 18 15 13 11 02/30 Read block 495 len 8192 03/30 Write block 343 len 40960 data: 16 17 18 19 14 14 14 14 12 15 04/30 Read block 103 len 28672 05/30 Write block 262 len 40960 data: 14 13 11 9 13 17 13 15 15 17 06/30 Read block 251 len 24576 07/30 Read block 299 len 20480 08/30 Write block 534 len 24576 data: 17 14 15 15 16 14 09/30 Read block 520 len 8192 10/30 Read block 183 len 28672 11/30 Read block 301 len 40960 12/30 Write block 355 len 16384 data: 17 15 13 12 13/30 Read block 4 len 12288 14/30 Write block 86 len 36864 data: 11 8 8 12 12 10 12 13 11 15/30 Read block 167 len 40960 16/30 Flush 17/30 Flush 18/30 Write block 531 len 20480 data: 16 17 18 18 15 19/30 Flush 20/30 Write block 501 len 32768 data: 14 11 15 19 23 22 24 21 21/30 Read block 367 len 24576 22/30 Write block 143 len 28672 data: 11 10 9 9 11 13 13 23/30 Read block 146 len 24576 24/30 Write block 143 len 8192 data: 12 11 25/30 Write block 255 len 36864 data: 18 17 16 14 15 15 13 15 14 26/30 Write block 219 len 20480 data: 12 11 12 14 13 27/30 Write block 346 len 36864 data: 20 15 15 15 15 13 16 17 16 28/30 Read block 569 len 12288 29/30 Read block 122 len 32768 30/30 Write block 17 len 24576 data: 13 11 12 9 9 7 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:30:58.568287001Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1910} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 243 243 243 F T T <--- 1 020-039 155 155 155 246 246 246 F T T <--- 2 040-059 156 158 158 248 250 250 T F F <--- 3 060-079 157 157 157 249 249 249 F F F 4 080-099 157 158 158 249 250 250 F F F <--- 5 100-119 156 156 156 248 248 248 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 152 152 152 243 243 243 F T T <--- 8 160-179 157 157 157 249 249 249 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 156 156 156 248 248 248 F T T <--- 11 220-239 157 157 157 249 249 249 F T T <--- 12 240-259 155 155 155 246 246 246 F T T <--- 13 260-279 155 158 158 246 250 250 F T T <--- 14 280-299 155 155 155 246 246 246 F F F 15 300-319 155 155 155 246 246 246 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 156 158 158 248 250 250 F T T <--- 18 360-379 157 157 157 249 249 249 F F F 19 380-399 156 156 156 248 248 248 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 157 157 157 249 249 249 F F F 22 440-459 157 157 157 249 249 249 F F F 23 460-479 155 155 155 246 246 246 F F F 24 480-499 157 157 157 249 249 249 F F F 25 500-519 156 156 156 248 248 248 F T T <--- 26 520-539 146 158 158 232 252 252 F F F <--- 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 158, Max flush: 252 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 253 253 253 F F F 1 020-039 159 159 159 253 253 253 F F F 2 040-059 158 158 158 250 250 250 F F F 3 060-079 157 157 157 249 249 249 F F F 4 080-099 158 158 158 250 250 250 F F F 5 100-119 156 156 156 248 248 248 F F F 6 120-139 152 152 152 242 242 242 F F F 7 140-159 159 159 159 253 253 253 F F F 8 160-179 157 157 157 249 249 249 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 159 159 159 253 253 253 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 159 159 159 253 253 253 F F F 13 260-279 159 159 159 253 253 253 F F F 14 280-299 155 155 155 246 246 246 F F F 15 300-319 155 155 155 246 246 246 F F F 16 320-339 148 148 148 237 237 237 F F F 17 340-359 159 159 159 253 253 253 F F F 18 360-379 157 157 157 249 249 249 F F F 19 380-399 156 156 156 248 248 248 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 157 157 157 249 249 249 F F F 22 440-459 157 157 157 249 249 249 F F F 23 460-479 155 155 155 246 246 246 F F F 24 480-499 157 157 157 249 249 249 F F F 25 500-519 159 159 159 253 253 253 F F F 26 520-539 158 158 158 252 252 252 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 159, Max flush: 253 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.901070124Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.912853365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91317113Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91320716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913220364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91323891Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"Crucible 8aa787b6-e899-4beb-b799-fa070e146a04 has session id: 07fae1b9-9d0a-4209-8032-032a6093da22","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913601238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Upstairs opts: Upstairs UUID: 8aa787b6-e899-4beb-b799-fa070e146a04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913636038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Crucible stats registered with UUID: 8aa787b6-e899-4beb-b799-fa070e146a04","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913651473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:12.913665077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913817622Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913861485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91387661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.913995485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91401074Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"8aa787b6-e899-4beb-b799-fa070e146a04 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914031846Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914047621Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914061615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914075389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914089003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914103487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914123303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"ds_connection connected from Ok(127.0.0.1:35481)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914293574Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"ds_connection connected from Ok(127.0.0.1:56877)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914321772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"ds_connection connected from Ok(127.0.0.1:34860)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914337287Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91495111Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.914989712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915397593Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91572177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[0]R flush_numbers[0..12]: [253, 253, 250, 249, 250, 248, 242, 253, 249, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915745877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[0]R generation[0..12]: [159, 159, 158, 157, 158, 156, 152, 159, 157, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915762462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915782748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[1]R flush_numbers[0..12]: [253, 253, 250, 249, 250, 248, 242, 253, 249, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915799143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[1]R generation[0..12]: [159, 159, 158, 157, 158, 156, 152, 159, 157, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915815027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915831552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[2]R flush_numbers[0..12]: [253, 253, 250, 249, 250, 248, 242, 253, 249, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915847357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[2]R generation[0..12]: [159, 159, 158, 157, 158, 156, 152, 159, 157, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915863122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915879637Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915893251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-17T14:31:12.915907645Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915926911Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915940535Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"":"downstairs","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915954489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915967293Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"0","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915981377Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"1","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.915997452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"client":"2","":"downstairs","session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.916011866Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.916038024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"8aa787b6-e899-4beb-b799-fa070e146a04 is now active with session: 07fae1b9-9d0a-4209-8032-032a6093da22","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.916055669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.916068993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919,"session_id":"07fae1b9-9d0a-4209-8032-032a6093da22"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.916082227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} {"msg":"Activated sub_volume 8aa787b6-e899-4beb-b799-fa070e146a04","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:12.91609369Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 45 len 12288 data: 11 13 17 02/30 Write block 119 len 8192 data: 12 12 03/30 Flush 04/30 Write block 171 len 4096 data: 8 05/30 Write block 100 len 36864 data: 15 14 17 14 14 15 15 15 10 06/30 Read block 571 len 32768 07/30 Write block 357 len 12288 data: 14 13 10 08/30 Read block 107 len 8192 09/30 Read block 352 len 8192 10/30 Read block 147 len 32768 11/30 Read block 64 len 36864 12/30 Write block 424 len 32768 data: 12 12 10 13 13 10 11 13 13/30 Flush 14/30 Flush 15/30 Write block 432 len 28672 data: 12 14 13 15 14 12 10 16/30 Write block 494 len 24576 data: 13 11 7 7 7 9 17/30 Flush 18/30 Write block 315 len 28672 data: 11 13 14 11 13 13 13 19/30 Read block 0 len 28672 20/30 Read block 407 len 20480 21/30 Read block 58 len 20480 22/30 Write block 385 len 28672 data: 16 13 14 13 12 10 12 23/30 Read block 434 len 16384 24/30 Read block 523 len 16384 25/30 Write block 122 len 36864 data: 14 11 12 11 11 11 13 11 8 26/30 Write block 72 len 8192 data: 11 11 27/30 Write block 4 len 28672 data: 7 9 11 11 12 13 12 28/30 Write block 471 len 20480 data: 9 11 10 8 10 29/30 Write block 293 len 28672 data: 12 15 13 15 13 16 18 30/30 Read block 420 len 40960 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 25 27 Done 27 25 2 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:12.928378871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1919} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 253 253 253 T T F <--- 1 020-039 159 159 159 253 253 253 F F F 2 040-059 160 160 160 254 254 254 F F F 3 060-079 157 157 157 249 249 249 T T F <--- 4 080-099 158 158 158 250 250 250 F F F 5 100-119 160 160 160 255 255 254 F F T <--- 6 120-139 160 160 160 254 254 254 T T F <--- 7 140-159 159 159 159 253 253 253 F F F 8 160-179 160 160 157 255 255 249 F F T <--- 9 180-199 155 155 155 246 246 246 F F F 10 200-219 159 159 159 253 253 253 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 159 159 159 253 253 253 F F F 13 260-279 159 159 159 253 253 253 F F F 14 280-299 155 155 155 246 246 246 T T F <--- 15 300-319 155 155 155 246 246 246 T T F <--- 16 320-339 148 148 148 237 237 237 T T F <--- 17 340-359 160 160 159 255 255 253 F F F <--- 18 360-379 157 157 157 249 249 249 F F F 19 380-399 156 156 156 248 248 248 T T F <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 160 160 157 257 257 249 F F F <--- 22 440-459 157 157 157 249 249 249 F F F 23 460-479 155 155 155 246 246 246 T T F <--- 24 480-499 160 160 157 257 257 249 F F F <--- 25 500-519 159 159 159 253 253 253 F F F 26 520-539 158 158 158 252 252 252 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 160, Max flush: 257 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 258 258 258 F F F 1 020-039 159 159 159 253 253 253 F F F 2 040-059 160 160 160 254 254 254 F F F 3 060-079 161 161 161 258 258 258 F F F 4 080-099 158 158 158 250 250 250 F F F 5 100-119 160 160 160 255 255 255 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 159 159 159 253 253 253 F F F 8 160-179 160 160 160 255 255 255 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 159 159 159 253 253 253 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 159 159 159 253 253 253 F F F 13 260-279 159 159 159 253 253 253 F F F 14 280-299 161 161 161 258 258 258 F F F 15 300-319 161 161 161 258 258 258 F F F 16 320-339 161 161 161 258 258 258 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 157 157 157 249 249 249 F F F 19 380-399 161 161 161 258 258 258 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 160 160 160 257 257 257 F F F 22 440-459 157 157 157 249 249 249 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 160 160 160 257 257 257 F F F 25 500-519 159 159 159 253 253 253 F F F 26 520-539 158 158 158 252 252 252 F F F 27 540-559 155 155 155 246 246 246 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F F F Max gen: 161, Max flush: 258 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.263037884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:27.273845664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.273891528Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.284274741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.284563117Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.284602859Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.284614673Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.284632348Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"Crucible 7a8a197f-acb6-492b-a3c8-a264c1572670 has session id: 78ba29fb-2afa-4b24-814c-838416f55d70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285002128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Upstairs opts: Upstairs UUID: 7a8a197f-acb6-492b-a3c8-a264c1572670, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28504163Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Crucible stats registered with UUID: 7a8a197f-acb6-492b-a3c8-a264c1572670","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285057364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:27.285071129Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285232157Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285269348Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285284942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285408299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285422873Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"7a8a197f-acb6-492b-a3c8-a264c1572670 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285436497Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285449231Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285463085Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28547691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285523944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28554704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285562175Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"ds_connection connected from Ok(127.0.0.1:37119)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285587242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"ds_connection connected from Ok(127.0.0.1:51333)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28567903Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"ds_connection connected from Ok(127.0.0.1:44757)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.285706508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286322201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28635168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286368885Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286761712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[0]R flush_numbers[0..12]: [258, 253, 254, 258, 250, 255, 258, 253, 255, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286783799Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[0]R generation[0..12]: [161, 159, 160, 161, 158, 160, 161, 159, 160, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286800174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286816759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[1]R flush_numbers[0..12]: [258, 253, 254, 258, 250, 255, 258, 253, 255, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286832313Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[1]R generation[0..12]: [161, 159, 160, 161, 158, 160, 161, 159, 160, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286847788Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286863933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[2]R flush_numbers[0..12]: [258, 253, 254, 258, 250, 255, 258, 253, 255, 246, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286879848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[2]R generation[0..12]: [161, 159, 160, 161, 158, 160, 161, 159, 160, 155, 159, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286898613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286915718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286929312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286943306Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg"Wait for a query_work_queue command to finish before sending IO :"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286962892Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286976576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"":"downstairs","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.286991011Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287003904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"0","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287017689Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"1","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287032153Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"client":"2","":"downstairs","session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287046427Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287059741Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"7a8a197f-acb6-492b-a3c8-a264c1572670 is now active with session: 78ba29fb-2afa-4b24-814c-838416f55d70","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287077616Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.28709033Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928,"session_id":"78ba29fb-2afa-4b24-814c-838416f55d70"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287103694Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} {"msg":"Activated sub_volume 7a8a197f-acb6-492b-a3c8-a264c1572670","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:27.287115268Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 32768 data: 11 11 12 10 13 14 13 14 02/30 Read block 455 len 24576 03/30 Read block 159 len 36864 04/30 Read block 244 len 4096 05/30 Read block 103 len 20480 06/30 Write block 539 len 36864 data: 15 10 14 13 12 14 13 13 12 07/30 Write block 528 len 28672 data: 12 11 12 17 18 19 19 08/30 Write block 60 len 40960 data: 12 13 11 11 11 12 12 15 18 16 09/30 Read block 539 len 28672 10/30 Write block 105 len 16384 data: 16 16 16 11 11/30 Write block 296 len 32768 data: 16 14 17 19 18 15 14 13 12/30 Read block 49 len 40960 13/30 Read block 439 len 32768 14/30 Write block 367 len 36864 data: 6 8 10 11 14 13 18 18 17 15/30 Read block 273 len 32768 16/30 Read block 567 len 12288 17/30 Write block 18 len 24576 data: 12 13 10 10 8 8 18/30 Write block 581 len 4096 data: 14 19/30 Read block 452 len 4096 20/30 Read block 110 len 36864 21/30 Write block 26 len 12288 data: 9 11 11 22/30 Write block 505 len 16384 data: 24 23 25 22 23/30 Read block 382 len 32768 24/30 Write block 200 len 40960 data: 14 12 12 12 12 10 10 10 12 12 25/30 Write block 4 len 12288 data: 8 10 12 26/30 Write block 442 len 28672 data: 7 11 15 16 15 10 11 27/30 Write block 242 len 32768 data: 13 12 12 11 12 12 16 18 28/30 Read block 87 len 8192 29/30 Write block 147 len 4096 data: 12 30/30 Read block 346 len 20480 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:27.299469529Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 258 258 258 F T T <--- 1 020-039 159 159 159 253 253 253 F T T <--- 2 040-059 160 160 160 254 254 254 F F F 3 060-079 161 161 161 258 258 258 F T T <--- 4 080-099 158 158 158 250 250 250 F F F 5 100-119 160 160 160 255 255 255 F T T <--- 6 120-139 161 161 161 258 258 258 F F F 7 140-159 159 159 159 253 253 253 F T T <--- 8 160-179 160 160 160 255 255 255 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 159 159 159 253 253 253 F T T <--- 11 220-239 159 159 159 253 253 253 F F F 12 240-259 159 159 159 253 253 253 F T T <--- 13 260-279 159 159 159 253 253 253 F F F 14 280-299 161 161 161 258 258 258 F T T <--- 15 300-319 161 161 161 258 258 258 F T T <--- 16 320-339 161 161 161 258 258 258 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 157 157 157 249 249 249 F T T <--- 19 380-399 161 161 161 258 258 258 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 160 160 160 257 257 257 F F F 22 440-459 157 157 157 249 249 249 F T T <--- 23 460-479 161 161 161 258 258 258 F F F 24 480-499 160 160 160 257 257 257 T T T 25 500-519 159 159 159 253 253 253 F T T <--- 26 520-539 158 158 158 252 252 252 F T T <--- 27 540-559 155 155 155 246 246 246 F T T <--- 28 560-579 148 148 148 237 237 237 F F F 29 580-599 153 153 153 244 244 244 F T T <--- Max gen: 161, Max flush: 258 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 162 162 162 259 259 259 F F F 2 040-059 160 160 160 254 254 254 F F F 3 060-079 162 162 162 259 259 259 F F F 4 080-099 158 158 158 250 250 250 F F F 5 100-119 162 162 162 259 259 259 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 162 162 162 259 259 259 F F F 8 160-179 160 160 160 255 255 255 F F F 9 180-199 155 155 155 246 246 246 F F F 10 200-219 162 162 162 259 259 259 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 162 162 162 259 259 259 F F F 13 260-279 159 159 159 253 253 253 F F F 14 280-299 162 162 162 259 259 259 F F F 15 300-319 162 162 162 259 259 259 F F F 16 320-339 161 161 161 258 258 258 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 162 162 162 259 259 259 F F F 19 380-399 161 161 161 258 258 258 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 160 160 160 257 257 257 F F F 22 440-459 162 162 162 259 259 259 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 162 162 162 259 259 259 F F F 25 500-519 162 162 162 259 259 259 F F F 26 520-539 162 162 162 259 259 259 F F F 27 540-559 162 162 162 259 259 259 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 162 162 162 259 259 259 F F F Max gen: 162, Max flush: 259 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.645774581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:41.656481681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.656520623Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.666762575Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667057392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667097024Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667108838Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667126823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"Crucible 636772d1-65f2-4ec9-b8dd-7e4817f3984d has session id: 106c5712-405d-4201-9638-739523473f43","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667460403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Upstairs opts: Upstairs UUID: 636772d1-65f2-4ec9-b8dd-7e4817f3984d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.6674871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Crucible stats registered with UUID: 636772d1-65f2-4ec9-b8dd-7e4817f3984d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667501855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:41.667515379Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667680738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667704835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667722861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667852939Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667870145Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"636772d1-65f2-4ec9-b8dd-7e4817f3984d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.66788716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667899813Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667914168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667928152Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.667963622Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668006485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668032113Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"ds_connection connected from Ok(127.0.0.1:44192)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668048208Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"ds_connection connected from Ok(127.0.0.1:61142)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668063502Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"ds_connection connected from Ok(127.0.0.1:43901)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668078327Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668739724Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668765041Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.668885057Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669254417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[0]R flush_numbers[0..12]: [259, 259, 254, 259, 250, 259, 258, 259, 255, 246, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669280325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 162, 158, 162, 161, 162, 160, 155, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.66929637Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669312995Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[1]R flush_numbers[0..12]: [259, 259, 254, 259, 250, 259, 258, 259, 255, 246, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669329159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 162, 158, 162, 161, 162, 160, 155, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669344834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669360819Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[2]R flush_numbers[0..12]: [259, 259, 254, 259, 250, 259, 258, 259, 255, 246, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669376333Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 162, 158, 162, 161, 162, 160, 155, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669392218Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669408723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Max found gen is 163","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669422267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Generation requested: 164 >= found:163","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669436661Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669455347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669471172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"":"downstairs","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669485916Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.66949883Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"0","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669512834Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"1","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669527168Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"client":"2","":"downstairs","session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669541683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669557877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"636772d1-65f2-4ec9-b8dd-7e4817f3984d is now active with session: 106c5712-405d-4201-9638-739523473f43","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669573082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669586056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937,"session_id":"106c5712-405d-4201-9638-739523473f43"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.66959938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} {"msg":"Activated sub_volume 636772d1-65f2-4ec9-b8dd-7e4817f3984d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:41.669611003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 8192 data: 13 11 02/30 Write block 489 len 28672 data: 14 15 14 15 12 14 12 03/30 Write block 434 len 40960 data: 14 16 15 13 11 8 4 4 8 12 04/30 Write block 80 len 24576 data: 10 10 10 11 11 10 05/30 Write block 539 len 28672 data: 16 11 15 14 13 15 14 06/30 Read block 406 len 36864 07/30 Write block 323 len 12288 data: 12 11 12 08/30 Read block 203 len 8192 09/30 Flush 10/30 Write block 583 len 16384 data: 15 12 12 13 11/30 Read block 64 len 12288 12/30 Read block 556 len 8192 13/30 Write block 196 len 12288 data: 19 19 19 14/30 Write block 363 len 8192 data: 11 11 15/30 Read block 161 len 4096 16/30 Read block 342 len 12288 17/30 Read block 84 len 28672 18/30 Read block 474 len 24576 19/30 Write block 545 len 40960 data: 15 14 13 13 12 13 13 13 13 15 20/30 Flush 21/30 Write block 317 len 36864 data: 15 12 14 14 14 12 13 12 13 22/30 Write block 247 len 8192 data: 13 17 23/30 Write block 94 len 20480 data: 12 12 12 15 13 24/30 Read block 566 len 12288 25/30 Read block 328 len 40960 26/30 Read block 121 len 40960 27/30 Read block 407 len 12288 28/30 Read block 201 len 16384 29/30 Write block 286 len 8192 data: 13 14 30/30 Read block 118 len 8192 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 0 0 23 Done 7 30 30 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:41.681406968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1937} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 162 162 162 259 259 259 F F F 2 040-059 160 160 160 254 254 254 F F F 3 060-079 162 162 162 259 259 259 F F F 4 080-099 158 164 164 250 260 260 T T T <--- 5 100-119 162 162 162 259 259 259 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 162 162 162 259 259 259 F F F 8 160-179 160 160 160 255 255 255 F F F 9 180-199 155 164 164 246 261 261 F F F <--- 10 200-219 162 162 162 259 259 259 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 162 162 162 259 259 259 F T T <--- 13 260-279 159 159 159 253 253 253 F F F 14 280-299 162 162 162 259 259 259 F T T <--- 15 300-319 162 162 162 259 259 259 F T T <--- 16 320-339 161 164 164 258 260 260 T T T <--- 17 340-359 160 160 160 255 255 255 F F F 18 360-379 162 164 164 259 261 261 F F F <--- 19 380-399 161 164 164 258 260 260 T F F <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 160 164 164 257 260 260 T F F <--- 22 440-459 162 164 164 259 260 260 T F F <--- 23 460-479 161 161 161 258 258 258 F F F 24 480-499 162 164 164 259 260 260 T F F <--- 25 500-519 162 162 162 259 259 259 F F F 26 520-539 162 164 164 259 260 260 T F F <--- 27 540-559 162 164 164 259 261 261 T F F <--- 28 560-579 148 148 148 237 237 237 F F F 29 580-599 162 164 164 259 261 261 F F F <--- Max gen: 164, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 162 162 162 259 259 259 F F F 2 040-059 160 160 160 254 254 254 F F F 3 060-079 162 162 162 259 259 259 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 162 162 162 259 259 259 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 162 162 162 259 259 259 F F F 8 160-179 160 160 160 255 255 255 F F F 9 180-199 164 164 164 261 261 261 F F F 10 200-219 162 162 162 259 259 259 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 165 165 165 262 262 262 F F F 13 260-279 159 159 159 253 253 253 F F F 14 280-299 165 165 165 262 262 262 F F F 15 300-319 165 165 165 262 262 262 F F F 16 320-339 165 165 165 262 262 262 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 164 164 164 261 261 261 F F F 19 380-399 164 164 164 260 260 260 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 164 164 164 260 260 260 F F F 22 440-459 164 164 164 260 260 260 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 162 162 162 259 259 259 F F F 26 520-539 164 164 164 260 260 260 F F F 27 540-559 164 164 164 261 261 261 F F F 28 560-579 148 148 148 237 237 237 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 165, Max flush: 262 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.017118275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.028880179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029223522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029262423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029274647Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029295943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"Crucible d8cdbd2f-9ff8-48cd-8540-971da9f86544 has session id: a63ad7da-4ff0-4756-9897-53bbb136684d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029701874Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Upstairs opts: Upstairs UUID: d8cdbd2f-9ff8-48cd-8540-971da9f86544, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029737795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Crucible stats registered with UUID: d8cdbd2f-9ff8-48cd-8540-971da9f86544","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.02975362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:56.029767284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029919519Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029946707Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.029961732Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030108825Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.03012436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"d8cdbd2f-9ff8-48cd-8540-971da9f86544 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030139565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030152508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030166963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030180847Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030194451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030210976Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.03022572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60948)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030272624Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65531)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030300933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36546)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030316877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"io task","client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030956938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.030989808Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031293238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031649034Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[0]R flush_numbers[0..12]: [259, 259, 254, 259, 262, 259, 258, 259, 255, 261, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031671831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 162, 165, 162, 161, 162, 160, 164, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031688256Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031704831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[1]R flush_numbers[0..12]: [259, 259, 254, 259, 262, 259, 258, 259, 255, 261, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031721146Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 162, 165, 162, 161, 162, 160, 164, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.03173674Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031756646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[2]R flush_numbers[0..12]: [259, 259, 254, 259, 262, 259, 258, 259, 255, 261, 259, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031772591Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 162, 165, 162, 161, 162, 160, 164, 162, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031788506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031804791Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031818405Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:31:56.031832399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031851615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031865159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"":"downstairs","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031879123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031891697Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"0","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031905511Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"1","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031926337Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"client":"2","":"downstairs","session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031940851Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031954365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"d8cdbd2f-9ff8-48cd-8540-971da9f86544 is now active with session: a63ad7da-4ff0-4756-9897-53bbb136684d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031971961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031986345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946,"session_id":"a63ad7da-4ff0-4756-9897-53bbb136684d"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.031999789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} {"msg":"Activated sub_volume d8cdbd2f-9ff8-48cd-8540-971da9f86544","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:31:56.032011582Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 269 len 28672 data: 16 16 18 22 22 16 14 02/30 Read block 95 len 16384 03/30 Write block 146 len 40960 data: 10 13 14 14 11 9 10 12 9 9 04/30 Read block 198 len 16384 05/30 Read block 490 len 40960 06/30 Read block 384 len 28672 07/30 Read block 95 len 24576 08/30 Write block 556 len 36864 data: 17 16 16 14 15 17 16 18 16 09/30 Read block 571 len 12288 10/30 Read block 48 len 24576 11/30 Read block 425 len 24576 12/30 Read block 494 len 4096 13/30 Write block 426 len 12288 data: 11 14 14 14/30 Write block 34 len 24576 data: 12 11 11 10 10 9 15/30 Read block 291 len 24576 16/30 Write block 150 len 12288 data: 12 10 11 17/30 Write block 430 len 36864 data: 12 14 13 15 15 17 16 14 12 18/30 Write block 502 len 20480 data: 12 16 20 25 24 19/30 Write block 52 len 40960 data: 11 11 11 12 12 12 11 10 13 14 20/30 Write block 453 len 16384 data: 13 8 9 12 21/30 Read block 409 len 28672 22/30 Write block 278 len 40960 data: 10 11 13 10 9 10 12 14 14 15 23/30 Write block 208 len 28672 data: 13 13 11 11 8 9 10 24/30 Write block 511 len 12288 data: 18 17 15 25/30 Read block 461 len 20480 26/30 Flush 27/30 Write block 325 len 40960 data: 14 12 12 10 9 9 10 7 8 10 28/30 Read block 552 len 4096 29/30 Write block 147 len 36864 data: 14 15 15 13 11 12 13 10 10 30/30 Write block 158 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 28 3 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:31:56.045106284Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1946} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 162 166 263 259 263 F F F <--- 2 040-059 166 160 166 263 254 263 F F F <--- 3 060-079 166 162 166 263 259 263 F F F <--- 4 080-099 165 165 165 262 262 262 F F F 5 100-119 162 162 162 259 259 259 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 166 162 166 263 259 263 T T T <--- 8 160-179 160 160 160 255 255 255 F F F 9 180-199 164 164 164 261 261 261 F F F 10 200-219 166 162 166 263 259 263 F F F <--- 11 220-239 159 159 159 253 253 253 F F F 12 240-259 165 165 165 262 262 262 F F F 13 260-279 166 159 166 263 253 263 F T F <--- 14 280-299 166 165 166 263 262 263 F F F <--- 15 300-319 165 165 165 262 262 262 F F F 16 320-339 165 165 165 262 262 262 T F T <--- 17 340-359 160 160 160 255 255 255 F F F 18 360-379 164 164 164 261 261 261 F F F 19 380-399 164 164 164 260 260 260 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 164 166 263 260 263 F F F <--- 22 440-459 166 164 166 263 260 263 F F F <--- 23 460-479 161 161 161 258 258 258 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 166 162 166 263 259 263 F F F <--- 26 520-539 164 164 164 260 260 260 F F F 27 540-559 166 164 166 263 261 263 F F F <--- 28 560-579 166 148 166 263 237 263 F F F <--- 29 580-599 164 164 164 261 261 261 F F F Max gen: 166, Max flush: 263 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 166 166 166 263 263 263 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 162 162 162 259 259 259 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 167 167 167 264 264 264 F F F 8 160-179 160 160 160 255 255 255 F F F 9 180-199 164 164 164 261 261 261 F F F 10 200-219 166 166 166 263 263 263 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 165 165 165 262 262 262 F F F 13 260-279 166 166 166 263 263 263 F F F 14 280-299 166 166 166 263 263 263 F F F 15 300-319 165 165 165 262 262 262 F F F 16 320-339 167 167 167 264 264 264 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 164 164 164 261 261 261 F F F 19 380-399 164 164 164 260 260 260 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 F F F 22 440-459 166 166 166 263 263 263 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 166 166 166 263 263 263 F F F 26 520-539 164 164 164 260 260 260 F F F 27 540-559 166 166 166 263 263 263 F F F 28 560-579 166 166 166 263 263 263 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 167, Max flush: 264 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.37390658Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.385613908Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.385952109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386007375Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386020309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386038174Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"Crucible c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff has session id: 59cc3f4c-a5d8-497a-b653-aea1ccf344c9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386385608Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Upstairs opts: Upstairs UUID: c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386420568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Crucible stats registered with UUID: c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386436463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:10.386449877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386623409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386658389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386673594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386803362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386820157Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386833101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386852387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386866801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386880946Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.38689465Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386946475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386971483Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51149)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.386994209Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63806)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.387150166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37896)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.387173563Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.387707002Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.387726297Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.387826277Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388160937Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[0]R flush_numbers[0..12]: [259, 263, 263, 263, 262, 259, 258, 264, 255, 261, 263, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388183374Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[0]R generation[0..12]: [162, 166, 166, 166, 165, 162, 161, 167, 160, 164, 166, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388199448Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388215823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[1]R flush_numbers[0..12]: [259, 263, 263, 263, 262, 259, 258, 264, 255, 261, 263, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388231728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[1]R generation[0..12]: [162, 166, 166, 166, 165, 162, 161, 167, 160, 164, 166, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388247643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388263988Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[2]R flush_numbers[0..12]: [259, 263, 263, 263, 262, 259, 258, 264, 255, 261, 263, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388279642Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[2]R generation[0..12]: [162, 166, 166, 166, 165, 162, 161, 167, 160, 164, 166, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388295137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388311192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388324466Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.38833867Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388352014Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.38837373Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"":"downstairs","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388387775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388400518Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"0","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388414563Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"1","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388428787Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"client":"2","":"downstairs","session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388443261Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388456855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff is now active with session: 59cc3f4c-a5d8-497a-b653-aea1ccf344c9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.38847192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388484433Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955,"session_id":"59cc3f4c-a5d8-497a-b653-aea1ccf344c9"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388497917Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} {"msg":"Activated sub_volume c81740ba-50cf-4bfa-9ebd-7fb6c0ae91ff","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:10.388509351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 174 len 8192 data: 10 13 02/30 Read block 503 len 20480 03/30 Read block 170 len 36864 04/30 Write block 392 len 16384 data: 12 10 13 9 05/30 Write block 381 len 24576 data: 14 12 9 10 17 14 06/30 Read block 176 len 12288 07/30 Write block 155 len 20480 data: 11 11 12 13 12 08/30 Read block 529 len 12288 09/30 Write block 202 len 20480 data: 13 13 13 11 11 10/30 Read block 174 len 28672 11/30 Read block 0 len 16384 12/30 Read block 216 len 40960 13/30 Read block 144 len 4096 14/30 Read block 531 len 32768 15/30 Write block 113 len 16384 data: 20 17 14 11 16/30 Read block 595 len 12288 17/30 Write block 46 len 20480 data: 14 18 19 16 14 18/30 Read block 437 len 32768 19/30 Read block 144 len 8192 20/30 Write block 280 len 24576 data: 14 11 10 11 13 15 21/30 Read block 270 len 24576 22/30 Write block 551 len 40960 data: 14 14 14 16 16 18 17 17 15 16 23/30 Write block 182 len 8192 data: 9 8 24/30 Read block 451 len 32768 25/30 Read block 263 len 20480 26/30 Read block 450 len 32768 27/30 Write block 193 len 20480 data: 12 13 13 20 20 28/30 Read block 74 len 8192 29/30 Write block 242 len 36864 data: 14 13 13 12 13 14 18 19 18 30/30 Read block 355 len 36864 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:10.401172394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1955} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 166 166 166 263 263 263 T F T <--- 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 162 162 162 259 259 259 T F T <--- 6 120-139 161 161 161 258 258 258 F F F 7 140-159 167 167 167 264 264 264 T F T <--- 8 160-179 160 160 160 255 255 255 T T T 9 180-199 164 164 164 261 261 261 T F T <--- 10 200-219 166 166 166 263 263 263 T F T <--- 11 220-239 159 159 159 253 253 253 F F F 12 240-259 165 165 165 262 262 262 T F T <--- 13 260-279 166 166 166 263 263 263 F F F 14 280-299 166 166 166 263 263 263 T F T <--- 15 300-319 165 165 165 262 262 262 F F F 16 320-339 167 167 167 264 264 264 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 164 164 164 261 261 261 F F F 19 380-399 164 164 164 260 260 260 T F T <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 F F F 22 440-459 166 166 166 263 263 263 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 166 166 166 263 263 263 F F F 26 520-539 164 164 164 260 260 260 F F F 27 540-559 166 166 166 263 263 263 T F T <--- 28 560-579 166 166 166 263 263 263 T F T <--- 29 580-599 164 164 164 261 261 261 F F F Max gen: 167, Max flush: 264 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 168 168 168 265 265 265 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 168 168 168 265 265 265 F F F 8 160-179 168 168 168 265 265 265 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 168 168 168 265 265 265 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 168 168 168 265 265 265 F F F 13 260-279 166 166 166 263 263 263 F F F 14 280-299 168 168 168 265 265 265 F F F 15 300-319 165 165 165 262 262 262 F F F 16 320-339 167 167 167 264 264 264 F F F 17 340-359 160 160 160 255 255 255 F F F 18 360-379 164 164 164 261 261 261 F F F 19 380-399 168 168 168 265 265 265 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 F F F 22 440-459 166 166 166 263 263 263 F F F 23 460-479 161 161 161 258 258 258 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 166 166 166 263 263 263 F F F 26 520-539 164 164 164 260 260 260 F F F 27 540-559 168 168 168 265 265 265 F F F 28 560-579 168 168 168 265 265 265 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 168, Max flush: 265 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.733273587Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745088797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745402691Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745441292Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745457387Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745475723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"Crucible 370e900d-8959-470f-88a5-9eaaf26e4756 has session id: 1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745771281Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Upstairs opts: Upstairs UUID: 370e900d-8959-470f-88a5-9eaaf26e4756, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745863398Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Crucible stats registered with UUID: 370e900d-8959-470f-88a5-9eaaf26e4756","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.745879773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:24.745893177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746059006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746094397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746109722Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746256875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74627235Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"370e900d-8959-470f-88a5-9eaaf26e4756 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746288665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746301609Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746315763Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746329817Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746343321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746361106Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746375391Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55730)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746427626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48027)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746455415Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56460)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.746471279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747186352Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747225544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747242269Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747593264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[0]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 265, 265, 265, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74761648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[0]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 168, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747632475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74764906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[1]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 265, 265, 265, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747665075Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[1]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 168, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74768097Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747699795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[2]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 265, 265, 265, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.7477162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[2]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 168, 168, 168, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747732305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74774846Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747761844Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74778123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747795064Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747808708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"":"downstairs","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747822822Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747835566Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"0","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.7478498Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"1","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.74788296Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"client":"2","":"downstairs","session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747900255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747913649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"370e900d-8959-470f-88a5-9eaaf26e4756 is now active with session: 1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747928423Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747943098Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964,"session_id":"1a4e8e8a-3296-4bcf-82c3-cc893d3b18d6"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747956502Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} {"msg":"Activated sub_volume 370e900d-8959-470f-88a5-9eaaf26e4756","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:24.747967895Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 4096 data: 15 02/30 Write block 172 len 28672 data: 9 8 11 14 16 13 11 03/30 Read block 136 len 16384 04/30 Read block 246 len 40960 05/30 Flush 06/30 Write block 349 len 28672 data: 16 16 14 17 18 17 18 07/30 Read block 299 len 8192 08/30 Read block 414 len 24576 09/30 Write block 258 len 40960 data: 15 16 16 14 16 15 12 10 14 18 10/30 Read block 151 len 8192 11/30 Write block 464 len 40960 data: 10 12 14 15 13 11 8 10 12 11 12/30 Write block 210 len 16384 data: 12 12 9 10 13/30 Write block 562 len 36864 data: 17 19 17 15 17 17 15 15 13 14/30 Write block 370 len 8192 data: 12 15 15/30 Read block 498 len 32768 16/30 Read block 221 len 24576 17/30 Write block 301 len 32768 data: 16 15 14 13 10 9 8 12 18/30 Write block 314 len 20480 data: 9 12 14 16 13 19/30 Write block 548 len 40960 data: 14 13 14 15 15 15 17 17 19 18 20/30 Read block 533 len 28672 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 470 len 12288 data: 9 11 13 25/30 Read block 506 len 16384 26/30 Read block 139 len 16384 27/30 Read block 42 len 36864 28/30 Write block 514 len 28672 data: 14 12 12 15 13 9 8 29/30 Write block 509 len 36864 data: 18 19 19 18 16 15 13 13 16 30/30 Read block 382 len 8192 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 26 28 Done 30 28 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:24.761253224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1964} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 168 168 168 265 265 265 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 168 168 168 265 265 265 F F F 8 160-179 170 170 168 266 266 265 F F T <--- 9 180-199 168 168 168 265 265 265 F F F 10 200-219 170 170 168 267 267 265 F F F <--- 11 220-239 159 159 159 253 253 253 F F F 12 240-259 170 170 168 267 267 265 F F F <--- 13 260-279 170 170 166 267 267 263 F F F <--- 14 280-299 168 168 168 265 265 265 F F F 15 300-319 170 170 165 267 267 262 F F F <--- 16 320-339 167 167 167 264 264 264 F F F 17 340-359 170 170 160 267 267 255 F F F <--- 18 360-379 170 170 164 267 267 261 F F F <--- 19 380-399 170 170 168 266 266 265 F F T <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 F F F 22 440-459 166 166 166 263 263 263 F F F 23 460-479 170 170 161 267 267 258 T T F <--- 24 480-499 164 164 164 260 260 260 F F F 25 500-519 166 166 166 263 263 263 T T F <--- 26 520-539 164 164 164 260 260 260 T T F <--- 27 540-559 170 170 168 267 267 265 F F F <--- 28 560-579 170 170 168 267 267 265 F F F <--- 29 580-599 164 164 164 261 261 261 F F F Max gen: 170, Max flush: 267 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 259 259 259 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 168 168 168 265 265 265 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 168 168 168 265 265 265 F F F 8 160-179 170 170 170 266 266 266 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 170 170 170 267 267 267 F F F 11 220-239 159 159 159 253 253 253 F F F 12 240-259 170 170 170 267 267 267 F F F 13 260-279 170 170 170 267 267 267 F F F 14 280-299 168 168 168 265 265 265 F F F 15 300-319 170 170 170 267 267 267 F F F 16 320-339 167 167 167 264 264 264 F F F 17 340-359 170 170 170 267 267 267 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 F F F 22 440-459 166 166 166 263 263 263 F F F 23 460-479 171 171 171 268 268 268 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 171 171 171 268 268 268 F F F 26 520-539 171 171 171 268 268 268 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 171, Max flush: 268 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.092582354Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.104222132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.104528743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.104567105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.104579038Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.104597264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"Crucible ee3b3eaa-267c-49ef-a124-57ee6c0808b7 has session id: 108b2d55-5f91-4536-bc05-bdc48c9281c1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10495546Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Upstairs opts: Upstairs UUID: ee3b3eaa-267c-49ef-a124-57ee6c0808b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105001234Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Crucible stats registered with UUID: ee3b3eaa-267c-49ef-a124-57ee6c0808b7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105017469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:39.105031173Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10518961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105227882Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105243556Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105375176Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10539078Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"ee3b3eaa-267c-49ef-a124-57ee6c0808b7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105406755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105419479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105433393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105447227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105461131Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105475425Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10548959Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63281)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105577666Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46674)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105667913Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45993)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.105696191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"io task","client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.106266491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.106298491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.106660118Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107037681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[0]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 266, 265, 267, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107063279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[0]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 170, 168, 170, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107079663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107096138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[1]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 266, 265, 267, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107111973Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[1]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 170, 168, 170, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107127568Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107143943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[2]R flush_numbers[0..12]: [259, 263, 265, 263, 262, 265, 258, 265, 266, 265, 267, 253]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107159617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[2]R generation[0..12]: [162, 166, 168, 166, 165, 168, 161, 168, 170, 168, 170, 159]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107175142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107191737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107205251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107219455Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107241452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107255226Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"":"downstairs","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10727025Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107283364Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"0","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107297708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"1","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107311322Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"client":"2","":"downstairs","session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107325527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-03-17T14:32:39.107341701Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"ee3b3eaa-267c-49ef-a124-57ee6c0808b7 is now active with session: 108b2d55-5f91-4536-bc05-bdc48c9281c1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107356906Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.10736957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973,"session_id":"108b2d55-5f91-4536-bc05-bdc48c9281c1"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107383134Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} {"msg":"Activated sub_volume ee3b3eaa-267c-49ef-a124-57ee6c0808b7","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:39.107394967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 460 len 32768 data: 11 12 14 11 11 13 15 16 02/30 Write block 520 len 16384 data: 9 7 8 8 03/30 Flush 04/30 Write block 296 len 12288 data: 17 15 18 05/30 Flush 06/30 Write block 456 len 28672 data: 13 13 10 10 12 13 15 07/30 Read block 216 len 28672 08/30 Write block 214 len 4096 data: 11 09/30 Write block 222 len 12288 data: 15 14 12 10/30 Flush 11/30 Write block 455 len 16384 data: 10 14 14 11 12/30 Flush 13/30 Write block 4 len 8192 data: 9 11 14/30 Read block 132 len 8192 15/30 Write block 281 len 36864 data: 12 11 12 14 16 15 16 16 14 16/30 Read block 208 len 24576 17/30 Read block 390 len 36864 18/30 Flush 19/30 Flush 20/30 Write block 329 len 36864 data: 10 10 11 8 9 11 7 7 7 21/30 Write block 271 len 4096 data: 19 22/30 Read block 250 len 36864 23/30 Write block 140 len 12288 data: 14 13 13 24/30 Read block 452 len 20480 25/30 Read block 239 len 40960 26/30 Write block 433 len 8192 data: 16 16 27/30 Read block 563 len 16384 28/30 Read block 407 len 12288 29/30 Read block 500 len 20480 30/30 Read block 87 len 40960 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:39.119941576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1973} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 162 172 273 259 273 F F F <--- 1 020-039 166 166 166 263 263 263 F F F 2 040-059 168 168 168 265 265 265 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 168 168 168 265 265 265 T F T <--- 8 160-179 170 170 170 266 266 266 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 172 170 172 271 267 271 F F F <--- 11 220-239 172 159 172 271 253 271 F F F <--- 12 240-259 170 170 170 267 267 267 F F F 13 260-279 170 170 170 267 267 267 T F T <--- 14 280-299 172 168 172 273 265 273 F F F <--- 15 300-319 170 170 170 267 267 267 F F F 16 320-339 167 167 167 264 264 264 T F T <--- 17 340-359 170 170 170 267 267 267 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 166 166 166 263 263 263 T F T <--- 22 440-459 172 166 172 272 263 272 F F F <--- 23 460-479 172 171 172 271 268 271 F F F <--- 24 480-499 164 164 164 260 260 260 F F F 25 500-519 171 171 171 268 268 268 F F F 26 520-539 172 171 172 269 268 269 F F F <--- 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 172, Max flush: 273 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 273 273 273 F F F 1 020-039 166 166 166 263 263 263 F F F 2 040-059 168 168 168 265 265 265 F F F 3 060-079 166 166 166 263 263 263 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 161 161 161 258 258 258 F F F 7 140-159 173 173 173 274 274 274 F F F 8 160-179 170 170 170 266 266 266 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 172 172 172 271 271 271 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 170 170 170 267 267 267 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 172 172 273 273 273 F F F 15 300-319 170 170 170 267 267 267 F F F 16 320-339 173 173 173 274 274 274 F F F 17 340-359 170 170 170 267 267 267 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 173 173 173 274 274 274 F F F 22 440-459 172 172 172 272 272 272 F F F 23 460-479 172 172 172 271 271 271 F F F 24 480-499 164 164 164 260 260 260 F F F 25 500-519 171 171 171 268 268 268 F F F 26 520-539 172 172 172 269 269 269 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 173, Max flush: 274 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.44534182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457066494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457372625Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457421049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457444726Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457466743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"Crucible b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2 has session id: c93e8358-a3eb-4883-931a-fe4d5d530dfd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.45779347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Upstairs opts: Upstairs UUID: b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457832482Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Crucible stats registered with UUID: b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.457848196Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:53.45786162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458053078Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458087368Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458102912Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458218657Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458235822Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458253597Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458266211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458280615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.45829487Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458308544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458326179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458340463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36977)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458355668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:39563)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458378194Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57547)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.458393069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.459115164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.459163898Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.459640941Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460033027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[0]R flush_numbers[0..12]: [273, 263, 265, 263, 262, 265, 258, 274, 266, 265, 271, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460059905Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[0]R generation[0..12]: [172, 166, 168, 166, 165, 168, 161, 173, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460077441Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460094326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[1]R flush_numbers[0..12]: [273, 263, 265, 263, 262, 265, 258, 274, 266, 265, 271, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460110471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[1]R generation[0..12]: [172, 166, 168, 166, 165, 168, 161, 173, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460126025Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460145981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[2]R flush_numbers[0..12]: [273, 263, 265, 263, 262, 265, 258, 274, 266, 265, 271, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460162146Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[2]R generation[0..12]: [172, 166, 168, 166, 165, 168, 161, 173, 170, 168, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460177871Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460194326Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:32:53.46020795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460227585Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460241089Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460254944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"":"downstairs","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460268958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460281572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"0","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460295656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"1","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.46030957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"client":"2","":"downstairs","session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","levelDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":30,"time":"2026-03-17T14:32:53.460326725Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460340169Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2 is now active with session: c93e8358-a3eb-4883-931a-fe4d5d530dfd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460355183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460369398Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982,"session_id":"c93e8358-a3eb-4883-931a-fe4d5d530dfd"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460382692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} {"msg":"Activated sub_volume b08e5c80-2eef-4ffb-9a04-8c2dedcf3bc2","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:32:53.460394235Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 334 len 16384 data: 12 8 8 8 03/30 Write block 58 len 20480 data: 12 11 14 15 12 04/30 Write block 201 len 24576 data: 13 14 14 14 12 12 05/30 Read block 369 len 40960 06/30 Write block 121 len 12288 data: 12 15 12 07/30 Write block 211 len 4096 data: 13 08/30 Write block 530 len 4096 data: 13 09/30 Read block 364 len 24576 10/30 Write block 420 len 36864 data: 10 11 11 13 13 13 12 15 15 11/30 Write block 478 len 36864 data: 12 14 14 13 12 11 11 12 12 12/30 Write block 344 len 20480 data: 18 19 21 16 16 13/30 Flush 14/30 Write block 145 len 24576 data: 10 11 15 16 16 14 15/30 Write block 327 len 4096 data: 13 16/30 Write block 527 len 20480 data: 12 13 12 14 18 17/30 Write block 242 len 36864 data: 15 14 14 13 14 15 19 20 19 18/30 Write block 22 len 36864 data: 9 9 7 7 10 12 12 13 14 19/30 Read block 173 len 16384 20/30 Flush 21/30 Write block 431 len 16384 data: 15 14 17 17 22/30 Read block 101 len 8192 23/30 Write block 511 len 24576 data: 20 19 17 16 14 14 24/30 Read block 72 len 36864 25/30 Read block 259 len 36864 26/30 Read block 96 len 4096 27/30 Read block 246 len 24576 28/30 Write block 516 len 4096 data: 15 29/30 Read block 97 len 40960 30/30 Write block 317 len 24576 data: 17 14 15 15 15 13 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1000)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:32:53.471565904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1982} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 273 273 273 F F F 1 020-039 174 166 174 277 263 277 F F F <--- 2 040-059 174 168 174 276 265 276 F F F <--- 3 060-079 174 166 174 276 263 276 F F F <--- 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 174 161 174 276 258 276 F F F <--- 7 140-159 174 173 174 277 274 277 F F F <--- 8 160-179 170 170 170 266 266 266 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 174 172 174 276 271 276 F F F <--- 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 170 174 277 267 277 F F F <--- 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 172 172 273 273 273 F F F 15 300-319 170 170 170 267 267 267 T F T <--- 16 320-339 174 173 174 277 274 277 T F T <--- 17 340-359 174 170 174 276 267 276 F F F <--- 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 174 173 174 276 274 276 T F T <--- 22 440-459 172 172 172 272 272 272 F F F 23 460-479 174 172 174 276 271 276 F F F <--- 24 480-499 174 164 174 276 260 276 F F F <--- 25 500-519 171 171 171 268 268 268 T F T <--- 26 520-539 174 172 174 277 269 277 F F F <--- 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 174, Max flush: 277 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 273 273 273 F F F 1 020-039 174 174 174 277 277 277 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 174 174 174 276 276 276 F F F 4 080-099 165 165 165 262 262 262 F F F 5 100-119 168 168 168 265 265 265 F F F 6 120-139 174 174 174 276 276 276 F F F 7 140-159 174 174 174 277 277 277 F F F 8 160-179 170 170 170 266 266 266 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 174 174 174 276 276 276 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 174 174 277 277 277 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 172 172 273 273 273 F F F 15 300-319 175 175 175 278 278 278 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 175 175 175 278 278 278 F F F 22 440-459 172 172 172 272 272 272 F F F 23 460-479 174 174 174 276 276 276 F F F 24 480-499 174 174 174 276 276 276 F F F 25 500-519 175 175 175 278 278 278 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F F F Max gen: 175, Max flush: 278 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.807912399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:07.818618049Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.818672565Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829189258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829482286Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829520797Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829533501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829551416Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"Crucible 094b05c6-6440-4bbb-b652-1da1dda9ce0f has session id: c640f700-a08e-4dbf-a5b7-79cf58f4bd4b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829920286Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Upstairs opts: Upstairs UUID: 094b05c6-6440-4bbb-b652-1da1dda9ce0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829955357Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Crucible stats registered with UUID: 094b05c6-6440-4bbb-b652-1da1dda9ce0f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.829971401Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:07.829988366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.83013438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830170101Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830185875Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830309652Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830324547Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"094b05c6-6440-4bbb-b652-1da1dda9ce0f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830342242Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830355266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.83036919Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830383584Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830397408Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830412053Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830426227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44603)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830564868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57130)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830690406Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50647)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.830718764Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"io task","client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831327065Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831357354Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831374359Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831786042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[0]R flush_numbers[0..12]: [273, 277, 276, 276, 262, 265, 276, 277, 266, 265, 276, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831810069Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[0]R generation[0..12]: [172, 174, 174, 174, 165, 168, 174, 174, 170, 168, 174, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831826154Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831842969Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[1]R flush_numbers[0..12]: [273, 277, 276, 276, 262, 265, 276, 277, 266, 265, 276, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831859004Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[1]R generation[0..12]: [172, 174, 174, 174, 165, 168, 174, 174, 170, 168, 174, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831874599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831891143Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[2]R flush_numbers[0..12]: [273, 277, 276, 276, 262, 265, 276, 277, 266, 265, 276, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831906928Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[2]R generation[0..12]: [172, 174, 174, 174, 165, 168, 174, 174, 170, 168, 174, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831922603Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831939018Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831952532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2026-03-17T14:33:07.831969017Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.831988172Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832001216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"":"downstairs","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.83201541Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832028304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"0","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832042529Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"1","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832056443Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"client":"2","":"downstairs","session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:33:07.832071037Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832086542Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"094b05c6-6440-4bbb-b652-1da1dda9ce0f is now active with session: c640f700-a08e-4dbf-a5b7-79cf58f4bd4b","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832101626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.83211442Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991,"session_id":"c640f700-a08e-4dbf-a5b7-79cf58f4bd4b"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832127604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} {"msg":"Activated sub_volume 094b05c6-6440-4bbb-b652-1da1dda9ce0f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:07.832138967Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 20480 data: 11 12 12 14 14 02/30 Read block 153 len 8192 03/30 Read block 343 len 20480 04/30 Read block 310 len 12288 05/30 Write block 8 len 36864 data: 13 14 13 13 14 15 12 14 14 06/30 Read block 100 len 12288 07/30 Read block 7 len 16384 08/30 Write block 435 len 28672 data: 18 17 15 13 9 5 5 09/30 Read block 80 len 32768 10/30 Flush 11/30 Write block 449 len 32768 data: 12 15 16 13 14 9 11 15 12/30 Write block 212 len 4096 data: 10 13/30 Read block 170 len 8192 14/30 Read block 262 len 12288 15/30 Read block 139 len 4096 16/30 Flush 17/30 Write block 445 len 40960 data: 17 16 11 12 13 16 17 14 15 10 18/30 Read block 70 len 16384 19/30 Read block 570 len 28672 20/30 Read block 214 len 32768 21/30 Read block 166 len 32768 22/30 Write block 584 len 4096 data: 13 23/30 Write block 96 len 32768 data: 13 16 14 12 16 15 18 15 24/30 Read block 124 len 12288 25/30 Read block 284 len 36864 26/30 Write block 390 len 40960 data: 12 13 13 11 14 10 11 12 14 14 27/30 Read block 134 len 12288 28/30 Write block 158 len 36864 data: 14 13 11 9 10 11 11 11 11 29/30 Read block 75 len 4096 30/30 Read block 495 len 8192 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 1 26 Done 5 30 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:07.845987544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":1991} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 176 176 273 279 279 T F F <--- 1 020-039 174 174 174 277 277 277 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 174 174 174 276 276 276 F F F 4 080-099 165 165 165 262 262 262 F T T <--- 5 100-119 168 168 168 265 265 265 F T T <--- 6 120-139 174 174 174 276 276 276 F F F 7 140-159 174 174 174 277 277 277 F T T <--- 8 160-179 170 170 170 266 266 266 F T T <--- 9 180-199 168 168 168 265 265 265 F F F 10 200-219 174 176 176 276 280 280 F F F <--- 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 174 174 277 277 277 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 172 172 273 273 273 F F F 15 300-319 175 175 175 278 278 278 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 170 170 170 266 266 266 F T T <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 175 176 176 278 279 279 T F F <--- 22 440-459 172 176 176 272 280 280 F T T <--- 23 460-479 174 174 174 276 276 276 F F F 24 480-499 174 174 174 276 276 276 F F F 25 500-519 175 175 175 278 278 278 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 164 164 164 261 261 261 F T T <--- Max gen: 176, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 279 279 279 F F F 1 020-039 174 174 174 277 277 277 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 174 174 174 276 276 276 F F F 4 080-099 177 177 177 281 281 281 F F F 5 100-119 177 177 177 281 281 281 F F F 6 120-139 174 174 174 276 276 276 F F F 7 140-159 177 177 177 281 281 281 F F F 8 160-179 177 177 177 281 281 281 F F F 9 180-199 168 168 168 265 265 265 F F F 10 200-219 176 176 176 280 280 280 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 174 174 277 277 277 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 172 172 273 273 273 F F F 15 300-319 175 175 175 278 278 278 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 177 177 177 281 281 281 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 174 174 174 276 276 276 F F F 24 480-499 174 174 174 276 276 276 F F F 25 500-519 175 175 175 278 278 278 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F F F 29 580-599 177 177 177 281 281 281 F F F Max gen: 177, Max flush: 281 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.172685594Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:22.183383062Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.183434077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.193804897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194106027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194146059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194158653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194176848Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"Crucible 19b01b1e-561a-4b6f-aafa-96294ebb307e has session id: 914ec30d-a96b-4ab8-90da-15adfe96078e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194504296Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Upstairs opts: Upstairs UUID: 19b01b1e-561a-4b6f-aafa-96294ebb307e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194538656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Crucible stats registered with UUID: 19b01b1e-561a-4b6f-aafa-96294ebb307e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194555421Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:22.194571456Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194714798Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194744077Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194759932Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.19488971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194905125Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"19b01b1e-561a-4b6f-aafa-96294ebb307e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.1949223Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194938015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194952649Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194966753Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.194980177Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.19502164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195036384Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62734)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195051759Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49744)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195066703Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42437)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195187109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"io task","client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195756789Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195820968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.195925389Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196266041Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[0]R flush_numbers[0..12]: [279, 277, 276, 276, 281, 281, 276, 281, 281, 265, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196292258Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[0]R generation[0..12]: [176, 174, 174, 174, 177, 177, 174, 177, 177, 168, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196309493Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196329489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[1]R flush_numbers[0..12]: [279, 277, 276, 276, 281, 281, 276, 281, 281, 265, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196345924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[1]R generation[0..12]: [176, 174, 174, 174, 177, 177, 174, 177, 177, 168, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196361539Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196378164Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[2]R flush_numbers[0..12]: [279, 277, 276, 276, 281, 281, 276, 281, 281, 265, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196393839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[2]R generation[0..12]: [176, 174, 174, 174, 177, 177, 174, 177, 177, 168, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196409413Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196425708Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196439142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196458648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196472122Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196485596Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"":"downstairs","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.19649942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196512224Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"0","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196526378Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"1","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196540552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"client":"2","":"downstairs","session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:33:22.196557137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196571642Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"19b01b1e-561a-4b6f-aafa-96294ebb307e is now active with session: 914ec30d-a96b-4ab8-90da-15adfe96078e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196586626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.19659947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000,"session_id":"914ec30d-a96b-4ab8-90da-15adfe96078e"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196613244Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} {"msg":"Activated sub_volume 19b01b1e-561a-4b6f-aafa-96294ebb307e","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:22.196624837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 28672 data: 12 12 13 13 16 19 17 02/30 Read block 518 len 24576 03/30 Flush 04/30 Write block 591 len 20480 data: 11 10 12 9 8 05/30 Write block 8 len 32768 data: 14 15 14 14 15 16 13 15 06/30 Flush 07/30 Write block 316 len 4096 data: 15 08/30 Flush 09/30 Write block 25 len 36864 data: 8 11 13 13 14 15 12 13 12 10/30 Write block 60 len 36864 data: 15 16 13 13 13 14 14 17 20 11/30 Read block 434 len 40960 12/30 Write block 504 len 20480 data: 21 26 25 26 23 13/30 Write block 134 len 8192 data: 11 14 14/30 Read block 192 len 40960 15/30 Read block 281 len 40960 16/30 Write block 181 len 8192 data: 8 10 17/30 Read block 110 len 4096 18/30 Write block 283 len 16384 data: 13 15 17 16 19/30 Write block 136 len 36864 data: 13 15 14 13 15 14 14 13 12 20/30 Flush 21/30 Write block 573 len 28672 data: 7 8 7 7 10 10 10 22/30 Write block 489 len 40960 data: 15 16 15 16 13 15 13 8 8 8 23/30 Read block 323 len 8192 24/30 Read block 511 len 20480 25/30 Read block 122 len 16384 26/30 Write block 109 len 40960 data: 13 16 17 16 21 18 15 12 11 11 27/30 Read block 429 len 8192 28/30 Read block 446 len 12288 29/30 Read block 546 len 36864 30/30 Write block 166 len 24576 data: 12 10 10 8 6 9 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:22.209816428Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2000} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 178 178 279 283 283 F F F <--- 1 020-039 174 178 178 277 285 285 F F F <--- 2 040-059 174 174 174 276 276 276 F F F 3 060-079 174 178 178 276 285 285 T F F <--- 4 080-099 177 177 177 281 281 281 F F F 5 100-119 177 177 177 281 281 281 F T T <--- 6 120-139 174 178 178 276 285 285 F F F <--- 7 140-159 177 178 178 281 285 285 F F F <--- 8 160-179 177 177 177 281 281 281 F T T <--- 9 180-199 168 178 178 265 285 285 F F F <--- 10 200-219 176 176 176 280 280 280 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 174 174 277 277 277 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 172 178 178 273 285 285 F F F <--- 15 300-319 175 178 178 278 284 284 F F F <--- 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 177 177 177 281 281 281 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 174 174 174 276 276 276 F F F 24 480-499 174 174 174 276 276 276 F T T <--- 25 500-519 175 178 178 278 285 285 F F F <--- 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 170 170 170 267 267 267 F T T <--- 29 580-599 177 178 178 281 283 283 F F F <--- Max gen: 178, Max flush: 285 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 178 178 178 285 285 285 F F F 4 080-099 177 177 177 281 281 281 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 178 178 178 285 285 285 F F F 8 160-179 179 179 179 286 286 286 F F F 9 180-199 178 178 178 285 285 285 F F F 10 200-219 176 176 176 280 280 280 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 174 174 277 277 277 F F F 13 260-279 173 173 173 274 274 274 F F F 14 280-299 178 178 178 285 285 285 F F F 15 300-319 178 178 178 284 284 284 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 170 170 170 267 267 267 F F F 19 380-399 177 177 177 281 281 281 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 174 174 174 276 276 276 F F F 24 480-499 179 179 179 286 286 286 F F F 25 500-519 178 178 178 285 285 285 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 179 179 179 286 286 286 F F F 29 580-599 178 178 178 283 283 283 F F F Max gen: 179, Max flush: 286 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.540537688Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:36.551287721Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.551333115Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.561571855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56185617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.561893251Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.561905625Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56192367Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"Crucible 3708b346-0a33-49c8-b856-af94e15207dd has session id: 1f3c3b2c-3185-4517-a6c0-7964212355ae","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562295621Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Upstairs opts: Upstairs UUID: 3708b346-0a33-49c8-b856-af94e15207dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562333742Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Crucible stats registered with UUID: 3708b346-0a33-49c8-b856-af94e15207dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562349567Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:36.562363651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562508785Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562533192Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562547776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56269588Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562711655Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"3708b346-0a33-49c8-b856-af94e15207dd active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56272872Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562741774Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562755868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562770092Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562784187Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562798651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562812885Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:43844)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562832201Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:43596)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.562851577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:52852)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.563016676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"io task","client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.563580664Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.563619055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.563725287Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564027667Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 276, 285, 281, 286, 285, 285, 286, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564050884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[0]R generation[0..12]: [178, 178, 174, 178, 177, 179, 178, 178, 179, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564066829Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564083634Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 276, 285, 281, 286, 285, 285, 286, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564099809Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[1]R generation[0..12]: [178, 178, 174, 178, 177, 179, 178, 178, 179, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564115643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564132068Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 276, 285, 281, 286, 285, 285, 286, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564148183Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[2]R generation[0..12]: [178, 178, 174, 178, 177, 179, 178, 178, 179, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564166779Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564183704Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564197198Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:33:36.564211262Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564230958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564244462Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"":"downstairs","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564258416Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56427125Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"0","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564285264Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"1","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564299658Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"client":"2","":"downstairs","session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564314513Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564330947Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"3708b346-0a33-49c8-b856-af94e15207dd is now active with session: 1f3c3b2c-3185-4517-a6c0-7964212355ae","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564346642Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564359976Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009,"session_id":"1f3c3b2c-3185-4517-a6c0-7964212355ae"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.56437355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} {"msg":"Activated sub_volume 3708b346-0a33-49c8-b856-af94e15207dd","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:36.564387014Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 40960 data: 10 12 12 13 8 7 9 11 13 16 02/30 Read block 52 len 32768 03/30 Read block 85 len 32768 04/30 Read block 390 len 8192 05/30 Read block 262 len 16384 06/30 Read block 521 len 40960 07/30 Read block 86 len 4096 08/30 Read block 511 len 28672 09/30 Write block 270 len 16384 data: 17 20 23 23 10/30 Flush 11/30 Flush 12/30 Flush 13/30 Write block 375 len 28672 data: 18 16 12 13 12 11 15 14/30 Read block 202 len 4096 15/30 Flush 16/30 Write block 259 len 28672 data: 17 17 15 17 16 13 11 17/30 Flush 18/30 Write block 252 len 36864 data: 13 12 17 19 18 17 16 18 18 19/30 Flush 20/30 Write block 70 len 4096 data: 15 21/30 Write block 165 len 24576 data: 12 13 11 11 9 7 22/30 Write block 170 len 24576 data: 8 10 10 9 12 15 23/30 Read block 536 len 20480 24/30 Write block 158 len 28672 data: 15 14 12 10 11 12 12 25/30 Write block 372 len 36864 data: 14 19 19 19 17 13 14 13 12 26/30 Write block 463 len 4096 data: 12 27/30 Read block 169 len 20480 28/30 Read block 549 len 24576 29/30 Read block 335 len 32768 30/30 Write block 495 len 12288 data: 14 9 9 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 18 1 1 20 Done 0 17 17 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:36.576853109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2009} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 178 178 178 285 285 285 F T T <--- 4 080-099 177 177 177 281 281 281 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 178 178 178 285 285 285 F T T <--- 8 160-179 179 179 179 286 286 286 F T T <--- 9 180-199 178 178 178 285 285 285 F F F 10 200-219 176 176 176 280 280 280 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 174 180 180 277 292 292 F F F <--- 13 260-279 180 180 180 287 292 292 F F F <--- 14 280-299 178 178 178 285 285 285 F F F 15 300-319 178 178 178 284 284 284 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 180 180 180 287 290 290 F T T <--- 19 380-399 177 180 180 281 290 290 F T T <--- 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 174 174 174 276 276 276 F T T <--- 24 480-499 179 179 179 286 286 286 F T T <--- 25 500-519 178 178 178 285 285 285 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 179 179 179 286 286 286 F F F 29 580-599 178 178 178 283 283 283 F F F Max gen: 180, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 F F F 2 040-059 174 174 174 276 276 276 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 177 177 177 281 281 281 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 181 181 181 293 293 293 F F F 8 160-179 181 181 181 293 293 293 F F F 9 180-199 178 178 178 285 285 285 F F F 10 200-219 176 176 176 280 280 280 F F F 11 220-239 172 172 172 271 271 271 F F F 12 240-259 180 180 180 292 292 292 F F F 13 260-279 180 180 180 292 292 292 F F F 14 280-299 178 178 178 285 285 285 F F F 15 300-319 178 178 178 284 284 284 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 181 181 181 293 293 293 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 181 181 181 293 293 293 F F F 24 480-499 181 181 181 293 293 293 F F F 25 500-519 178 178 178 285 285 285 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 F F F 28 560-579 179 179 179 286 286 286 F F F 29 580-599 178 178 178 283 283 283 F F F Max gen: 181, Max flush: 293 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.919779142Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.932320699Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.932715027Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.932761651Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.932773984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.93279204Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"Crucible ed470fb1-1c98-4055-be30-9569f2967ae5 has session id: 2c1e3e4b-7715-41d3-9f07-daa7c5498840","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933174904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Upstairs opts: Upstairs UUID: ed470fb1-1c98-4055-be30-9569f2967ae5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933242684Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Crucible stats registered with UUID: ed470fb1-1c98-4055-be30-9569f2967ae5","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933260499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:50.933273943Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933479485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933514355Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.93352993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.93369984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933719496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"ed470fb1-1c98-4055-be30-9569f2967ae5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933734861Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933747835Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.93376494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933778894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933792718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933807212Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933821096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"ds_connection connected from Ok(127.0.0.1:39101)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933927318Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"ds_connection connected from Ok(127.0.0.1:33346)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933953706Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"ds_connection connected from Ok(127.0.0.1:64658)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.933969221Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"io task","client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.934724005Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.934766348Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.934887404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935258935Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 276, 293, 281, 286, 285, 293, 293, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935287903Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[0]R generation[0..12]: [178, 178, 174, 181, 177, 179, 178, 181, 181, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935304178Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935320393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 276, 293, 281, 286, 285, 293, 293, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935339839Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[1]R generation[0..12]: [178, 178, 174, 181, 177, 179, 178, 181, 181, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935355644Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935372029Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 276, 293, 281, 286, 285, 293, 293, 285, 280, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935387963Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[2]R generation[0..12]: [178, 178, 174, 181, 177, 179, 178, 181, 181, 178, 176, 172]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935403868Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935420473Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935433987Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935448271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935461695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:33:50.935475509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"":"downstairs","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935493775Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935506629Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"0","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935520653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"1","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935534537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"client":"2","":"downstairs","session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935551222Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935565056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"ed470fb1-1c98-4055-be30-9569f2967ae5 is now active with session: 2c1e3e4b-7715-41d3-9f07-daa7c5498840","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935580351Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935592984Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018,"session_id":"2c1e3e4b-7715-41d3-9f07-daa7c5498840"} {"msg":"The guest has finished waiting for activation with:182"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935609489Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} {"msg":"Activated sub_volume ed470fb1-1c98-4055-be30-9569f2967ae5","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:33:50.935621193Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 32768 data: 11 12 9 9 13 13 11 13 02/30 Read block 41 len 28672 03/30 Write block 257 len 24576 data: 18 17 19 19 16 18 04/30 Write block 238 len 32768 data: 17 18 16 17 16 15 15 14 05/30 Read block 354 len 32768 06/30 Write block 250 len 12288 data: 20 13 14 07/30 Read block 400 len 32768 08/30 Flush 09/30 Write block 463 len 20480 data: 13 12 14 16 17 10/30 Read block 155 len 4096 11/30 Read block 423 len 8192 12/30 Write block 51 len 20480 data: 12 12 12 12 13 13/30 Write block 558 len 24576 data: 18 16 17 18 18 20 14/30 Read block 556 len 12288 15/30 Read block 25 len 40960 16/30 Write block 502 len 32768 data: 13 17 22 27 26 27 24 19 17/30 Write block 546 len 40960 data: 15 14 15 14 15 16 16 16 18 18 18/30 Write block 85 len 36864 data: 12 13 10 10 14 14 12 14 14 19/30 Read block 541 len 20480 20/30 Write block 143 len 36864 data: 14 13 11 12 16 17 17 15 12 21/30 Read block 42 len 4096 22/30 Write block 468 len 24576 data: 14 12 10 12 14 12 23/30 Write block 262 len 20480 data: 19 17 14 12 15 24/30 Read block 491 len 12288 25/30 Read block 350 len 12288 26/30 Read block 319 len 4096 27/30 Read block 92 len 4096 28/30 Read block 159 len 4096 29/30 Write block 85 len 12288 data: 13 14 11 30/30 Write block 208 len 32768 data: 14 14 13 14 11 11 12 10 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 23 25 Done 29 29 7 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:33:50.94876955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2018} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 F F F 2 040-059 174 174 174 276 276 276 T T F <--- 3 060-079 181 181 181 293 293 293 F F F 4 080-099 182 182 177 294 294 281 T T T <--- 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 181 181 181 293 293 293 T T F <--- 8 160-179 181 181 181 293 293 293 F F F 9 180-199 178 178 178 285 285 285 F F F 10 200-219 176 176 176 280 280 280 T T F <--- 11 220-239 182 182 172 294 294 271 F F T <--- 12 240-259 182 182 180 294 294 292 F F T <--- 13 260-279 182 182 180 294 294 292 T T T <--- 14 280-299 178 178 178 285 285 285 F F F 15 300-319 178 178 178 284 284 284 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 181 181 181 293 293 293 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 181 181 181 293 293 293 T T F <--- 24 480-499 181 181 181 293 293 293 F F F 25 500-519 178 178 178 285 285 285 T T F <--- 26 520-539 174 174 174 277 277 277 F F F 27 540-559 170 170 170 267 267 267 T T F <--- 28 560-579 179 179 179 286 286 286 T T F <--- 29 580-599 178 178 178 283 283 283 F F F Max gen: 182, Max flush: 294 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 F F F 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 183 183 295 295 295 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 183 183 183 295 295 295 F F F 8 160-179 181 181 181 293 293 293 F F F 9 180-199 178 178 178 285 285 285 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 182 182 182 294 294 294 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 183 183 183 295 295 295 F F F 14 280-299 178 178 178 285 285 285 F F F 15 300-319 178 178 178 284 284 284 F F F 16 320-339 175 175 175 278 278 278 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 181 181 181 293 293 293 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 183 183 183 295 295 295 F F F 24 480-499 181 181 181 293 293 293 F F F 25 500-519 183 183 183 295 295 295 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 183 183 183 295 295 295 F F F 28 560-579 183 183 183 295 295 295 F F F 29 580-599 178 178 178 283 283 283 F F F Max gen: 183, Max flush: 295 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.325610793Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.337328365Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.337633246Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.337670537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.33768281Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.337701186Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"Crucible 514c0bc7-688d-433b-9c94-faf605ead11a has session id: d73fdb89-4ff3-49c3-8155-7684fd12c578","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338067305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Upstairs opts: Upstairs UUID: 514c0bc7-688d-433b-9c94-faf605ead11a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338102636Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Crucible stats registered with UUID: 514c0bc7-688d-433b-9c94-faf605ead11a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338122151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:05.338135695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338298104Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338325942Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338340576Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.33848777Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338511957Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"514c0bc7-688d-433b-9c94-faf605ead11a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338524421Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338537485Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338552159Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338565864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338579698Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338593762Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338608006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"ds_connection connected from Ok(127.0.0.1:57982)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.33862275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"ds_connection connected from Ok(127.0.0.1:44964)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.338638105Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"ds_connection connected from Ok(127.0.0.1:53942)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.33865324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"io task","client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.339345676Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.33939278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.339985166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340316955Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[0]R flush_numbers[0..12]: [283, 285, 295, 293, 295, 286, 285, 295, 293, 285, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340340312Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[0]R generation[0..12]: [178, 178, 183, 181, 183, 179, 178, 183, 181, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340356887Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340373542Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[1]R flush_numbers[0..12]: [283, 285, 295, 293, 295, 286, 285, 295, 293, 285, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340389207Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[1]R generation[0..12]: [178, 178, 183, 181, 183, 179, 178, 183, 181, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340405382Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340421837Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[2]R flush_numbers[0..12]: [283, 285, 295, 293, 295, 286, 285, 295, 293, 285, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340437671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[2]R generation[0..12]: [178, 178, 183, 181, 183, 179, 178, 183, 181, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340453456Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340469671Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340486686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340506492Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340520096Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.34053343Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"":"downstairs","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340547644Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340560718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"0","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340575082Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"1","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340589066Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"client":"2","":"downstairs","session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:34:05.340603211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340618945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"514c0bc7-688d-433b-9c94-faf605ead11a is now active with session: d73fdb89-4ff3-49c3-8155-7684fd12c578","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.34063368Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340646523Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027,"session_id":"d73fdb89-4ff3-49c3-8155-7684fd12c578"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340659757Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} {"msg":"Activated sub_volume 514c0bc7-688d-433b-9c94-faf605ead11a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:05.340671311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 4096 data: 16 02/30 Read block 529 len 8192 03/30 Read block 198 len 24576 04/30 Read block 141 len 28672 05/30 Read block 437 len 8192 06/30 Flush 07/30 Write block 188 len 40960 data: 12 14 13 15 13 13 14 14 21 21 08/30 Read block 478 len 24576 09/30 Write block 503 len 36864 data: 18 23 28 27 28 25 20 20 21 10/30 Flush 11/30 Write block 495 len 32768 data: 15 10 10 9 10 14 15 14 12/30 Write block 565 len 20480 data: 16 18 18 16 16 13/30 Read block 174 len 16384 14/30 Read block 21 len 4096 15/30 Read block 184 len 40960 16/30 Read block 306 len 12288 17/30 Flush 18/30 Write block 319 len 36864 data: 16 16 16 14 14 13 15 13 14 19/30 Read block 21 len 36864 20/30 Read block 457 len 36864 21/30 Read block 556 len 32768 22/30 Read block 432 len 40960 23/30 Read block 21 len 4096 24/30 Write block 364 len 36864 data: 13 14 9 8 10 12 14 17 15 25/30 Read block 257 len 20480 26/30 Flush 27/30 Write block 299 len 20480 data: 20 19 17 16 15 28/30 Write block 36 len 12288 data: 12 11 11 29/30 Write block 289 len 32768 data: 15 14 12 12 13 16 14 18 30/30 Read block 437 len 40960 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:05.354034433Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2027} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 178 178 178 285 285 285 T F T <--- 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 183 183 295 295 295 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 183 183 183 295 295 295 F F F 8 160-179 181 181 181 293 293 293 F F F 9 180-199 184 178 184 297 285 297 F F F <--- 10 200-219 183 183 183 295 295 295 F F F 11 220-239 182 182 182 294 294 294 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 183 183 183 295 295 295 F F F 14 280-299 178 178 178 285 285 285 T F T <--- 15 300-319 184 178 184 299 284 299 T F T <--- 16 320-339 184 175 184 299 278 299 F F F <--- 17 340-359 174 174 174 276 276 276 F F F 18 360-379 184 181 184 299 293 299 F F F <--- 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 183 183 183 295 295 295 F F F 24 480-499 184 181 184 298 293 298 F F F <--- 25 500-519 184 183 184 298 295 298 F F F <--- 26 520-539 174 174 174 277 277 277 F F F 27 540-559 184 183 184 296 295 296 F T F <--- 28 560-579 184 183 184 298 295 298 F F F <--- 29 580-599 178 178 178 283 283 283 F F F Max gen: 184, Max flush: 299 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 185 185 185 300 300 300 F F F 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 183 183 295 295 295 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 183 183 183 295 295 295 F F F 8 160-179 181 181 181 293 293 293 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 182 182 182 294 294 294 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 183 183 183 295 295 295 F F F 14 280-299 185 185 185 300 300 300 F F F 15 300-319 185 185 185 300 300 300 F F F 16 320-339 184 184 184 299 299 299 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 184 184 184 299 299 299 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 176 176 176 279 279 279 F F F 22 440-459 177 177 177 281 281 281 F F F 23 460-479 183 183 183 295 295 295 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 184 184 184 298 298 298 F F F 26 520-539 174 174 174 277 277 277 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 184 184 184 298 298 298 F F F 29 580-599 178 178 178 283 283 283 F F F Max gen: 185, Max flush: 300 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.679819203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:19.690567135Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.69061738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701095102Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701387509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701426621Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701438814Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.70145681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"Crucible 84949f87-0080-4b52-a51d-9823773146f1 has session id: 64e70a1e-ade8-4a11-95ec-c766279ebf00","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701809515Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Upstairs opts: Upstairs UUID: 84949f87-0080-4b52-a51d-9823773146f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.701845305Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Crucible stats registered with UUID: 84949f87-0080-4b52-a51d-9823773146f1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.70186151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:19.701875054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702037803Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702073073Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702088548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702217886Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702232451Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"84949f87-0080-4b52-a51d-9823773146f1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702251336Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.70226436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702278445Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702292459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702306463Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702320827Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702335211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"ds_connection connected from Ok(127.0.0.1:63483)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702360829Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"ds_connection connected from Ok(127.0.0.1:53006)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702375904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"ds_connection connected from Ok(127.0.0.1:53299)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.702443544Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"io task","client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703072981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703120595Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703239031Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703569499Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[0]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 295, 293, 297, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703599908Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[0]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 183, 181, 184, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703619534Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703636399Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[1]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 295, 293, 297, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703652514Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[1]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 183, 181, 184, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703668339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703685124Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[2]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 295, 293, 297, 295, 294]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703700748Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[2]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 183, 181, 184, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703716743Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703733008Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703746412Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:34:19.703760626Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703779772Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703793156Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"":"downstairs","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.70380716Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703819924Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"0","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703836309Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"1","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703850303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"client":"2","":"downstairs","session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:34:19.703864537Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703880252Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"84949f87-0080-4b52-a51d-9823773146f1 is now active with session: 64e70a1e-ade8-4a11-95ec-c766279ebf00","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703895656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.7039087Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036,"session_id":"64e70a1e-ade8-4a11-95ec-c766279ebf00"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703922304Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} {"msg":"Activated sub_volume 84949f87-0080-4b52-a51d-9823773146f1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:19.703933768Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 438 len 36864 data: 14 10 6 6 9 13 16 18 17 02/30 Flush 03/30 Write block 574 len 16384 data: 9 8 8 11 04/30 Read block 576 len 28672 05/30 Read block 460 len 32768 06/30 Read block 286 len 8192 07/30 Write block 517 len 28672 data: 17 14 10 10 8 9 9 08/30 Read block 1 len 24576 09/30 Write block 369 len 28672 data: 13 15 18 16 20 20 20 10/30 Write block 156 len 28672 data: 12 13 16 15 13 11 12 11/30 Read block 22 len 24576 12/30 Write block 277 len 8192 data: 10 11 13/30 Read block 511 len 28672 14/30 Read block 596 len 12288 15/30 Read block 129 len 20480 16/30 Write block 266 len 20480 data: 16 19 14 17 18 17/30 Write block 223 len 20480 data: 15 13 13 15 13 18/30 Flush 19/30 Write block 147 len 16384 data: 17 18 18 16 20/30 Read block 194 len 40960 21/30 Read block 225 len 28672 22/30 Read block 533 len 12288 23/30 Read block 267 len 8192 24/30 Read block 419 len 28672 25/30 Read block 2 len 4096 26/30 Flush 27/30 Write block 236 len 4096 data: 16 28/30 Write block 560 len 32768 data: 18 19 19 21 18 17 19 19 29/30 Write block 440 len 40960 data: 7 7 10 14 17 19 18 12 13 14 30/30 Write block 594 len 24576 data: 10 9 7 6 6 5 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 2 2 29 Done 3 26 26 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:19.716464191Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2036} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 185 185 185 300 300 300 F F F 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 183 183 295 295 295 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 183 186 186 295 303 303 F F F <--- 8 160-179 181 186 186 293 302 302 F F F <--- 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 182 186 186 294 302 302 F T T <--- 12 240-259 182 182 182 294 294 294 F F F 13 260-279 183 186 186 295 302 302 F F F <--- 14 280-299 185 185 185 300 300 300 F F F 15 300-319 185 185 185 300 300 300 F F F 16 320-339 184 184 184 299 299 299 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 184 186 186 299 302 302 F F F <--- 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 186 186 186 301 301 301 F T T <--- 23 460-479 183 183 183 295 295 295 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 184 186 186 298 302 302 F F F <--- 26 520-539 174 186 186 277 302 302 F F F <--- 27 540-559 184 184 184 296 296 296 F F F 28 560-579 184 186 186 298 302 302 T T T <--- 29 580-599 178 178 178 283 283 283 F T T <--- Max gen: 186, Max flush: 303 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 185 185 185 300 300 300 F F F 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 183 183 295 295 295 F F F 5 100-119 179 179 179 286 286 286 F F F 6 120-139 178 178 178 285 285 285 F F F 7 140-159 186 186 186 303 303 303 F F F 8 160-179 186 186 186 302 302 302 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 187 187 187 304 304 304 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F F F 14 280-299 185 185 185 300 300 300 F F F 15 300-319 185 185 185 300 300 300 F F F 16 320-339 184 184 184 299 299 299 F F F 17 340-359 174 174 174 276 276 276 F F F 18 360-379 186 186 186 302 302 302 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 156 156 248 248 248 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 187 187 187 304 304 304 F F F 23 460-479 183 183 183 295 295 295 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 187 187 187 304 304 304 F F F Max gen: 187, Max flush: 304 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.048921876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:34.0596425Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.059700997Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070003267Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070290133Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070329394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070341728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070359773Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"Crucible b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85 has session id: 178c2a38-b3ec-49a1-97c1-e68e5945746f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070713279Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Upstairs opts: Upstairs UUID: b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070748999Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Crucible stats registered with UUID: b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070764744Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:34.070778338Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.0709184Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070942897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.070957641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071096233Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071111727Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071129723Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071142617Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071156661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071170635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071184889Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071199384Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071213738Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43095)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071270255Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44480)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071367494Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37702)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.071395522Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"io task","client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072052398Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072081076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072097211Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072507393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[0]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 303, 302, 297, 295, 304]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.0725303Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[0]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 186, 186, 184, 183, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072546755Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.07256311Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[1]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 303, 302, 297, 295, 304]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072579395Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[1]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 186, 186, 184, 183, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072595179Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072611434Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[2]R flush_numbers[0..12]: [283, 300, 295, 293, 295, 286, 285, 303, 302, 297, 295, 304]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072627059Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[2]R generation[0..12]: [178, 185, 183, 181, 183, 179, 178, 186, 186, 184, 183, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072645925Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072662449Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072676404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072690638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":Wait for a query_work_queue command to finish before sending IO 2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072710054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072723548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"":"downstairs","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072737712Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072750386Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"0","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.07276459Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"1","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072778394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"client":"2","":"downstairs","session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072793058Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072806662Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85 is now active with session: 178c2a38-b3ec-49a1-97c1-e68e5945746f","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072824278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072837112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045,"session_id":"178c2a38-b3ec-49a1-97c1-e68e5945746f"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072850475Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} {"msg":"Activated sub_volume b3db9d40-cf3d-4c5b-a8f3-27687f7e0f85","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:34.072864Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 8192 data: 16 11 02/30 Flush 03/30 Write block 318 len 12288 data: 15 17 17 04/30 Read block 518 len 8192 05/30 Read block 73 len 24576 06/30 Read block 416 len 16384 07/30 Read block 88 len 20480 08/30 Flush 09/30 Write block 476 len 16384 data: 12 11 13 15 10/30 Read block 87 len 12288 11/30 Write block 400 len 8192 data: 14 17 12/30 Write block 367 len 24576 data: 9 11 14 16 19 17 13/30 Flush 14/30 Write block 96 len 20480 data: 14 17 15 13 17 15/30 Read block 552 len 4096 16/30 Read block 121 len 36864 17/30 Read block 102 len 20480 18/30 Write block 91 len 36864 data: 13 15 15 13 13 15 18 16 14 19/30 Read block 369 len 24576 20/30 Write block 129 len 20480 data: 12 9 8 12 11 21/30 Read block 590 len 28672 22/30 Write block 84 len 24576 data: 12 14 15 12 11 15 23/30 Write block 224 len 12288 data: 14 14 16 24/30 Read block 428 len 32768 25/30 Write block 23 len 12288 data: 10 8 9 26/30 Flush 27/30 Write block 283 len 24576 data: 14 16 18 17 17 17 28/30 Read block 422 len 20480 29/30 Write block 459 len 20480 data: 11 13 14 16 14 30/30 Write block 334 len 40960 data: 13 9 9 9 9 11 10 12 14 17 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 2 33 Done 0 29 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:34.085162524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2045} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 185 188 188 300 308 308 F F F <--- 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 183 188 188 295 308 308 F F F <--- 5 100-119 179 188 188 286 308 308 F F F <--- 6 120-139 178 188 188 285 308 308 F F F <--- 7 140-159 186 186 186 303 303 303 F F F 8 160-179 186 186 186 302 302 302 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 187 188 188 304 308 308 F F F <--- 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F F F 14 280-299 185 185 185 300 300 300 F T T <--- 15 300-319 185 188 188 300 306 306 F F F <--- 16 320-339 184 188 188 299 306 306 F T T <--- 17 340-359 174 174 174 276 276 276 F T T <--- 18 360-379 186 188 188 302 307 307 F F F <--- 19 380-399 181 181 181 293 293 293 F F F 20 400-419 156 188 188 248 307 307 F F F <--- 21 420-439 186 186 186 301 301 301 F F F 22 440-459 187 188 188 304 305 305 F T T <--- 23 460-479 183 188 188 295 307 307 F T T <--- 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 187 187 187 304 304 304 F F F Max gen: 188, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 188 188 188 308 308 308 F F F 2 040-059 183 183 183 295 295 295 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 188 188 188 308 308 308 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 188 188 188 308 308 308 F F F 7 140-159 186 186 186 303 303 303 F F F 8 160-179 186 186 186 302 302 302 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F F F 14 280-299 189 189 189 309 309 309 F F F 15 300-319 188 188 188 306 306 306 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 189 189 189 309 309 309 F F F 18 360-379 188 188 188 307 307 307 F F F 19 380-399 181 181 181 293 293 293 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 189 189 189 309 309 309 F F F 23 460-479 189 189 189 309 309 309 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 187 187 187 304 304 304 F F F Max gen: 189, Max flush: 309 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.415290477Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427041638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427353711Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427391613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427403736Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427421452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"Crucible ac0ead81-b58b-4ea2-ab24-9a80eb8379c6 has session id: c320a90d-b8b3-4465-b5de-a6b561b75254","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427793692Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Upstairs opts: Upstairs UUID: ac0ead81-b58b-4ea2-ab24-9a80eb8379c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427830013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Crucible stats registered with UUID: ac0ead81-b58b-4ea2-ab24-9a80eb8379c6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.427845138Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:48.427858342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428023151Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428047438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428062353Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428218899Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428234774Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"ac0ead81-b58b-4ea2-ab24-9a80eb8379c6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428249758Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428262572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428276386Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.42829016Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428304135Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428318189Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428332653Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"ds_connection connected from Ok(127.0.0.1:32924)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428347758Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"ds_connection connected from Ok(127.0.0.1:58864)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428397032Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"ds_connection connected from Ok(127.0.0.1:36714)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.428424971Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"io task","client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.42906056Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.429105903Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.429605132Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.42993166Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[0]R flush_numbers[0..12]: [283, 308, 295, 293, 308, 308, 308, 303, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.429955417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[0]R generation[0..12]: [178, 188, 183, 181, 188, 188, 188, 186, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.429971811Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.429988526Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[1]R flush_numbers[0..12]: [283, 308, 295, 293, 308, 308, 308, 303, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430004481Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[1]R generation[0..12]: [178, 188, 183, 181, 188, 188, 188, 186, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430020216Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430036641Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[2]R flush_numbers[0..12]: [283, 308, 295, 293, 308, 308, 308, 303, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430052275Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[2]R generation[0..12]: [178, 188, 183, 181, 188, 188, 188, 186, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.43006782Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430083795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430097299Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:34:48.430111663Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430129638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430142552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"":"downstairs","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430156436Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430170981Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"0","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430185325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"1","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430198669Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"client":"2","":"downstairs","session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430213123Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430226527Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ac0ead81-b58b-4ea2-ab24-9a80eb8379c6 is now active with session: c320a90d-b8b3-4465-b5de-a6b561b75254","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430243042Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430256356Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054,"session_id":"c320a90d-b8b3-4465-b5de-a6b561b75254"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.43027006Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} {"msg":"Activated sub_volume ac0ead81-b58b-4ea2-ab24-9a80eb8379c6","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:34:48.430281334Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 12288 data: 16 14 13 02/30 Read block 113 len 28672 03/30 Read block 361 len 4096 04/30 Read block 441 len 40960 05/30 Read block 509 len 40960 06/30 Read block 99 len 20480 07/30 Flush 08/30 Write block 12 len 36864 data: 16 17 14 16 15 14 13 14 11 09/30 Read block 436 len 20480 10/30 Read block 265 len 20480 11/30 Write block 93 len 8192 data: 16 14 12/30 Read block 367 len 36864 13/30 Write block 48 len 4096 data: 20 14/30 Read block 98 len 16384 15/30 Read block 381 len 8192 16/30 Read block 202 len 36864 17/30 Read block 67 len 24576 18/30 Read block 7 len 28672 19/30 Read block 460 len 12288 20/30 Read block 538 len 24576 21/30 Write block 389 len 16384 data: 14 13 14 14 22/30 Flush 23/30 Write block 133 len 36864 data: 12 12 15 14 16 15 14 16 15 24/30 Flush 25/30 Flush 26/30 Flush 27/30 Write block 375 len 36864 data: 21 18 14 15 14 13 16 13 10 28/30 Read block 441 len 8192 29/30 Read block 104 len 4096 30/30 Write block 444 len 28672 data: 18 20 19 13 14 15 17 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:34:48.443241465Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2054} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 178 190 311 283 311 F F F <--- 1 020-039 190 188 190 311 308 311 F F F <--- 2 040-059 190 183 190 311 295 311 F F F <--- 3 060-079 181 181 181 293 293 293 F F F 4 080-099 190 188 190 311 308 311 F F F <--- 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 188 190 312 308 312 F F F <--- 7 140-159 190 186 190 312 303 312 F F F <--- 8 160-179 186 186 186 302 302 302 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F F F 14 280-299 189 189 189 309 309 309 F F F 15 300-319 188 188 188 306 306 306 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 189 189 189 309 309 309 F F F 18 360-379 188 188 188 307 307 307 T F T <--- 19 380-399 190 181 190 311 293 311 T F T <--- 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 189 189 189 309 309 309 T F T <--- 23 460-479 189 189 189 309 309 309 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 190 187 190 310 304 310 F T F <--- Max gen: 190, Max flush: 312 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 311 311 311 F F F 1 020-039 190 190 190 311 311 311 F F F 2 040-059 190 190 190 311 311 311 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 190 190 190 311 311 311 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 190 190 190 312 312 312 F F F 8 160-179 186 186 186 302 302 302 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 183 183 295 295 295 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F F F 14 280-299 189 189 189 309 309 309 F F F 15 300-319 188 188 188 306 306 306 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 189 189 189 309 309 309 F F F 18 360-379 191 191 191 313 313 313 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 191 191 191 313 313 313 F F F 23 460-479 189 189 189 309 309 309 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 191, Max flush: 313 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.767127687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:02.777807119Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.777859335Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.788292524Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.788572687Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.788610728Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.788622362Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.788639927Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"Crucible 97ce47ec-330b-49fa-a91e-56f6d9d3d2e9 has session id: 6e457d15-0a96-43d7-bdfd-e12a1eb4300a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789014269Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Upstairs opts: Upstairs UUID: 97ce47ec-330b-49fa-a91e-56f6d9d3d2e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789049719Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Crucible stats registered with UUID: 97ce47ec-330b-49fa-a91e-56f6d9d3d2e9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789064994Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:02.789078638Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789236855Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789260702Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789275396Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.78942013Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789434944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"97ce47ec-330b-49fa-a91e-56f6d9d3d2e9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.78945438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789467474Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789481438Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789495572Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789509606Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.78952404Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789540795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49273)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789648197Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36987)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789768683Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35086)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.789787339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"io task","client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79039339Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790457349Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790475944Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790839993Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 312, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79086543Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 190, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790881695Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79089801Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 312, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790913945Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 190, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790929709Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790946144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 312, 302, 297, 295, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.7909656Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 190, 186, 184, 183, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.790981435Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79099776Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791011144Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:35:02.791025228Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791043613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791056927Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"":"downstairs","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791071452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791084595Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"0","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79109902Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"1","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791113084Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"client":"2","":"downstairs","session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791127488Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791143823Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"97ce47ec-330b-49fa-a91e-56f6d9d3d2e9 is now active with session: 6e457d15-0a96-43d7-bdfd-e12a1eb4300a","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791158958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791171611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063,"session_id":"6e457d15-0a96-43d7-bdfd-e12a1eb4300a"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.791187496Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} {"msg":"Activated sub_volume 97ce47ec-330b-49fa-a91e-56f6d9d3d2e9","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:02.79119894Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 20480 data: 13 9 11 13 12 02/30 Read block 438 len 8192 03/30 Write block 351 len 8192 data: 15 18 04/30 Read block 318 len 32768 05/30 Read block 185 len 32768 06/30 Read block 198 len 4096 07/30 Read block 300 len 28672 08/30 Read block 223 len 28672 09/30 Write block 203 len 8192 data: 15 15 10/30 Read block 312 len 16384 11/30 Read block 433 len 4096 12/30 Write block 152 len 24576 data: 13 14 11 12 13 14 13/30 Read block 166 len 20480 14/30 Read block 5 len 36864 15/30 Read block 565 len 20480 16/30 Write block 157 len 12288 data: 15 17 16 17/30 Read block 587 len 36864 18/30 Read block 302 len 12288 19/30 Write block 161 len 4096 data: 12 20/30 Read block 325 len 8192 21/30 Read block 67 len 28672 22/30 Flush 23/30 Write block 268 len 32768 data: 15 18 19 21 24 24 17 15 24/30 Read block 132 len 28672 25/30 Read block 181 len 16384 26/30 Read block 504 len 8192 27/30 Read block 352 len 36864 28/30 Write block 305 len 20480 data: 11 10 9 13 12 29/30 Read block 233 len 20480 30/30 Read block 145 len 12288 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:02.803272366Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2063} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 311 311 311 F F F 1 020-039 190 190 190 311 311 311 F F F 2 040-059 190 190 190 311 311 311 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 190 190 190 311 311 311 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 190 192 192 312 314 314 F F F <--- 8 160-179 186 192 192 302 314 314 F F F <--- 9 180-199 184 184 184 297 297 297 F F F 10 200-219 183 192 192 295 314 314 F F F <--- 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 186 186 186 302 302 302 F T T <--- 14 280-299 189 189 189 309 309 309 F F F 15 300-319 188 188 188 306 306 306 F T T <--- 16 320-339 189 189 189 309 309 309 F F F 17 340-359 189 192 192 309 314 314 T F F <--- 18 360-379 191 191 191 313 313 313 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 191 191 191 313 313 313 F F F 23 460-479 189 192 192 309 314 314 T F F <--- 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 192, Max flush: 314 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 311 311 311 F F F 1 020-039 190 190 190 311 311 311 F F F 2 040-059 190 190 190 311 311 311 F F F 3 060-079 181 181 181 293 293 293 F F F 4 080-099 190 190 190 311 311 311 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 192 192 192 314 314 314 F F F 8 160-179 192 192 192 314 314 314 F F F 9 180-199 184 184 184 297 297 297 F F F 10 200-219 192 192 192 314 314 314 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 F F F 13 260-279 193 193 193 315 315 315 F F F 14 280-299 189 189 189 309 309 309 F F F 15 300-319 193 193 193 315 315 315 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 191 191 191 313 313 313 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 F F F 22 440-459 191 191 191 313 313 313 F F F 23 460-479 192 192 192 314 314 314 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 187 187 187 304 304 304 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 193, Max flush: 315 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.142707532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.154494094Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.154838447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.154877388Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.154889452Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.154907087Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"Crucible d0597843-8fe0-412d-b471-e4aaae43832d has session id: c8fad357-471d-4332-ab1e-54b682202da1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155226112Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Upstairs opts: Upstairs UUID: d0597843-8fe0-412d-b471-e4aaae43832d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155260733Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Crucible stats registered with UUID: d0597843-8fe0-412d-b471-e4aaae43832d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155276157Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:17.155289361Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155467604Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155492802Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155507686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.15565559Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155671055Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"d0597843-8fe0-412d-b471-e4aaae43832d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155686509Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155699393Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155713397Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155727222Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155740966Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.1557552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155780417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33816)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155795742Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40286)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155810726Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51798)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.155825371Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"io task","client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.156570933Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.156613686Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157105532Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157458858Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[0]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 314, 314, 297, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157482345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 192, 192, 184, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157498479Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157515044Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[1]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 314, 314, 297, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157531109Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 192, 192, 184, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157546884Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157563009Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[2]R flush_numbers[0..12]: [311, 311, 311, 293, 311, 308, 312, 314, 314, 297, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157578783Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 181, 190, 188, 190, 192, 192, 184, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157594508Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157610573Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157624137Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Generation requested: 194 >= found:194","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157643923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157657447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157670931Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"":"downstairs","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157684795Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157699029Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"0","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157713003Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"1","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157726897Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"client":"2","":"downstairs","session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157741092ZDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157757737Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"d0597843-8fe0-412d-b471-e4aaae43832d is now active with session: c8fad357-471d-4332-ab1e-54b682202da1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157772741Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157785295Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072,"session_id":"c8fad357-471d-4332-ab1e-54b682202da1"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157798769Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} {"msg":"Activated sub_volume d0597843-8fe0-412d-b471-e4aaae43832d","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:17.157810162Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 564 len 8192 data: 19 18 02/30 Read block 227 len 4096 03/30 Flush 04/30 Write block 294 len 20480 data: 17 15 19 16 19 05/30 Read block 238 len 8192 06/30 Read block 10 len 24576 07/30 Read block 87 len 36864 08/30 Read block 521 len 32768 09/30 Read block 324 len 8192 10/30 Write block 43 len 28672 data: 10 11 12 15 19 21 17 11/30 Write block 6 len 40960 data: 13 12 15 16 15 15 17 18 15 17 12/30 Write block 313 len 12288 data: 9 10 13 13/30 Read block 103 len 8192 14/30 Read block 37 len 40960 15/30 Read block 65 len 36864 16/30 Write block 21 len 32768 data: 11 10 11 9 10 12 14 14 17/30 Read block 31 len 4096 18/30 Flush 19/30 Write block 194 len 24576 data: 15 15 22 22 20 15 20/30 Read block 234 len 16384 21/30 Write block 58 len 28672 data: 13 12 16 17 14 14 14 22/30 Flush 23/30 Flush 24/30 Write block 4 len 36864 data: 10 12 14 13 16 17 16 16 18 25/30 Write block 280 len 12288 data: 15 13 12 26/30 Read block 211 len 20480 27/30 Write block 450 len 40960 data: 18 18 15 17 12 12 16 15 12 12 28/30 Write block 430 len 16384 data: 13 16 15 18 29/30 Write block 252 len 4096 data: 15 30/30 Read block 136 len 4096 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Read 1 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 4 Sent Done Sent false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 30 33 Done 27 30 0 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:17.170122681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2072} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 190 317 317 311 T T F <--- 1 020-039 194 194 190 317 317 311 F F F <--- 2 040-059 194 194 190 318 318 311 F F F <--- 3 060-079 194 194 181 318 318 293 F F F <--- 4 080-099 190 190 190 311 311 311 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 192 192 192 314 314 314 F F F 8 160-179 192 192 192 314 314 314 F F F 9 180-199 194 194 184 318 318 297 F F F <--- 10 200-219 192 192 192 314 314 314 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 182 182 182 294 294 294 T T F <--- 13 260-279 193 193 193 315 315 315 F F F 14 280-299 194 194 189 317 317 309 T T F <--- 15 300-319 194 194 193 317 317 315 F F F <--- 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 191 191 191 313 313 313 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 186 186 186 301 301 301 T T F <--- 22 440-459 191 191 191 313 313 313 T T F <--- 23 460-479 192 192 192 314 314 314 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 194 194 187 316 316 304 F F F <--- 29 580-599 190 190 190 310 310 310 F F F Max gen: 194, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 319 319 319 F F F 1 020-039 194 194 194 317 317 317 F F F 2 040-059 194 194 194 318 318 318 F F F 3 060-079 194 194 194 318 318 318 F F F 4 080-099 190 190 190 311 311 311 F F F 5 100-119 188 188 188 308 308 308 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 192 192 192 314 314 314 F F F 8 160-179 192 192 192 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 192 192 192 314 314 314 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 195 195 195 319 319 319 F F F 13 260-279 193 193 193 315 315 315 F F F 14 280-299 195 195 195 319 319 319 F F F 15 300-319 194 194 194 317 317 317 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 191 191 191 313 313 313 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 195 195 195 319 319 319 F F F 22 440-459 195 195 195 319 319 319 F F F 23 460-479 192 192 192 314 314 314 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 186 186 302 302 302 F F F 27 540-559 184 184 184 296 296 296 F F F 28 560-579 194 194 194 316 316 316 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 195, Max flush: 319 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.495308182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:31.506016643Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.506067938Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.516348611Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.516628265Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.516665116Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.516677599Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.516699506Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"Crucible eaffe182-8f01-4a1a-bcd2-82cfa3c32ece has session id: 38bf3efd-15df-471e-86af-650696cba582","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517036306Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Upstairs opts: Upstairs UUID: eaffe182-8f01-4a1a-bcd2-82cfa3c32ece, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517066665Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Crucible stats registered with UUID: eaffe182-8f01-4a1a-bcd2-82cfa3c32ece","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.51708214Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:31.517095804Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517253961Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517277768Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517292442Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517448409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517465754Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"eaffe182-8f01-4a1a-bcd2-82cfa3c32ece active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517499394Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517513668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517527923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517541877Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517555581Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517572236Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.51758648Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"ds_connection connected from Ok(127.0.0.1:39941)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517656661Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"ds_connection connected from Ok(127.0.0.1:55172)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.517737345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"ds_connection connected from Ok(127.0.0.1:52261)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.51775548Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"io task","client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518334923Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518371324Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518436363Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518764181Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[0]R flush_numbers[0..12]: [319, 317, 318, 318, 311, 308, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518787238Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[0]R generation[0..12]: [195, 194, 194, 194, 190, 188, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518803403Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518819958Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[1]R flush_numbers[0..12]: [319, 317, 318, 318, 311, 308, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518835812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[1]R generation[0..12]: [195, 194, 194, 194, 190, 188, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518851447Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518871103Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[2]R flush_numbers[0..12]: [319, 317, 318, 318, 311, 308, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518887128Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[2]R generation[0..12]: [195, 194, 194, 194, 190, 188, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518902812Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518918977Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518932501Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518952417Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518965831Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518979325Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"":"downstairs","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.518993469Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519006503Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"0","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519020507Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"1","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519034612Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"client":"2","":"downstairs","session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519049036Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519066491Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"eaffe182-8f01-4a1a-bcd2-82cfa3c32ece is now active with session: 38bf3efd-15df-471e-86af-650696cba582","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519081696Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.51909577Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081,"session_id":"38bf3efd-15df-471e-86af-650696cba582"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519109204Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} {"msg":"Activated sub_volume eaffe182-8f01-4a1a-bcd2-82cfa3c32ece","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:31.519120627Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 38 len 12288 data: 12 10 7 02/30 Read block 506 len 32768 03/30 Read block 62 len 16384 04/30 Read block 279 len 28672 05/30 Read block 200 len 12288 06/30 Write block 465 len 4096 data: 15 07/30 Flush 08/30 Write block 362 len 16384 data: 11 13 14 15 09/30 Read block 566 len 36864 10/30 Write block 64 len 8192 data: 15 15 11/30 Write block 529 len 28672 data: 13 15 19 19 20 20 16 12/30 Read block 147 len 12288 13/30 Read block 0 len 24576 14/30 Read block 524 len 24576 15/30 Read block 20 len 32768 16/30 Read block 51 len 40960 17/30 Read block 38 len 4096 18/30 Read block 297 len 28672 19/30 Flush 20/30 Flush 21/30 Write block 93 len 40960 data: 17 15 14 16 19 17 15 18 16 19 22/30 Write block 548 len 16384 data: 16 15 16 17 23/30 Read block 219 len 36864 24/30 Write block 279 len 8192 data: 12 16 25/30 Read block 469 len 28672 26/30 Write block 309 len 8192 data: 13 13 27/30 Read block 580 len 4096 28/30 Read block 524 len 32768 29/30 Write block 275 len 8192 data: 16 10 30/30 Write block 364 len 32768 data: 15 16 10 10 12 15 17 20 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 4 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:31.531317321Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2081} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 319 319 319 F F F 1 020-039 194 196 196 317 320 320 T F F <--- 2 040-059 194 196 196 318 320 320 T F F <--- 3 060-079 194 196 196 318 321 321 F F F <--- 4 080-099 190 190 190 311 311 311 F T T <--- 5 100-119 188 188 188 308 308 308 F T T <--- 6 120-139 190 190 190 312 312 312 F F F 7 140-159 192 192 192 314 314 314 F F F 8 160-179 192 192 192 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 192 192 192 314 314 314 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 195 195 195 319 319 319 F F F 13 260-279 193 193 193 315 315 315 F T T <--- 14 280-299 195 195 195 319 319 319 F T T <--- 15 300-319 194 194 194 317 317 317 F T T <--- 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 191 196 196 313 321 321 F T T <--- 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 195 195 195 319 319 319 F F F 22 440-459 195 195 195 319 319 319 F F F 23 460-479 192 196 196 314 320 320 F F F <--- 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 186 196 196 302 321 321 F F F <--- 27 540-559 184 184 184 296 296 296 F T T <--- 28 560-579 194 194 194 316 316 316 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 196, Max flush: 321 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 319 319 319 F F F 1 020-039 196 196 196 320 320 320 F F F 2 040-059 196 196 196 320 320 320 F F F 3 060-079 196 196 196 321 321 321 F F F 4 080-099 197 197 197 322 322 322 F F F 5 100-119 197 197 197 322 322 322 F F F 6 120-139 190 190 190 312 312 312 F F F 7 140-159 192 192 192 314 314 314 F F F 8 160-179 192 192 192 314 314 314 F F F 9 180-199 194 194 194 318 318 318 F F F 10 200-219 192 192 192 314 314 314 F F F 11 220-239 188 188 188 308 308 308 F F F 12 240-259 195 195 195 319 319 319 F F F 13 260-279 197 197 197 322 322 322 F F F 14 280-299 197 197 197 322 322 322 F F F 15 300-319 197 197 197 322 322 322 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 197 197 197 322 322 322 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 195 195 195 319 319 319 F F F 22 440-459 195 195 195 319 319 319 F F F 23 460-479 196 196 196 320 320 320 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 196 196 196 321 321 321 F F F 27 540-559 197 197 197 322 322 322 F F F 28 560-579 194 194 194 316 316 316 F F F 29 580-599 190 190 190 310 310 310 F F F Max gen: 197, Max flush: 322 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.861043083Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.872760054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873066045Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"cd46d0b314aa4fecb3f38344dd0644d67f94821c\",\n git_commit_timestamp: \"2026-03-16T19:55:04.000000000Z\",\n git_branch: \"mkeeter/min-quorum-negotiation-redux\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873103796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87311615Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873134555Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"Crucible 0c59c211-3359-4aa9-91d7-6035d01e7cb1 has session id: c6cfaafd-d370-4c2f-869a-ebecdf855185","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873510287Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Upstairs opts: Upstairs UUID: 0c59c211-3359-4aa9-91d7-6035d01e7cb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873544768Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Crucible stats registered with UUID: 0c59c211-3359-4aa9-91d7-6035d01e7cb1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873560392Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:45.873573646Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873728342Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873762822Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873781668Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873918089Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873933613Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"0c59c211-3359-4aa9-91d7-6035d01e7cb1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873949718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873962552Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873977076Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.873991471Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87402271Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87405563Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.874079227Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"ds_connection connected from Ok(127.0.0.1:43324)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87415858Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"ds_connection connected from Ok(127.0.0.1:44394)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.874276416Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"ds_connection connected from Ok(127.0.0.1:56007)","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.874293781Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"io task","client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.874967962Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875009904Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875419796Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875765409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[0]R flush_numbers[0..12]: [319, 320, 320, 321, 322, 322, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875790347Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[0]R generation[0..12]: [195, 196, 196, 196, 197, 197, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875806681Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875823266Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[1]R flush_numbers[0..12]: [319, 320, 320, 321, 322, 322, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875839331Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[1]R generation[0..12]: [195, 196, 196, 196, 197, 197, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875854876Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875871051Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[2]R flush_numbers[0..12]: [319, 320, 320, 321, 322, 322, 312, 314, 314, 318, 314, 308]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875887015Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[2]R generation[0..12]: [195, 196, 196, 196, 197, 197, 190, 192, 192, 194, 192, 188]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87590278Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875919345Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875933119Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-17T14:35:45.875947203Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875966409Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.875979893Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"":"downstairs","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876010182Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876023376Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"0","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.87603718Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"1","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876051054Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"client":"2","":"downstairs","session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-17T14:35:45.876065349Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876080033Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"0c59c211-3359-4aa9-91d7-6035d01e7cb1 is now active with session: c6cfaafd-d370-4c2f-869a-ebecdf855185","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876094907Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876107511Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090,"session_id":"c6cfaafd-d370-4c2f-869a-ebecdf855185"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876120635Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} {"msg":"Activated sub_volume 0c59c211-3359-4aa9-91d7-6035d01e7cb1","v":0,"name":"crucible","level":30,"time":"2026-03-17T14:35:45.876131968Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 16384 data: 18 17 11 11 02/30 Write block 173 len 36864 data: 10 13 16 17 14 12 13 11 9 03/30 Read block 132 len 24576 04/30 Read block 342 len 8192 05/30 Write block 541 len 16384 data: 16 15 14 16 06/30 Read block 68 len 24576 07/30 Read block 253 len 24576 08/30 Flush 09/30 Write block 17 len 36864 data: 15 14 15 12 12 11 12 10 11 10/30 Read block 16 len 8192 11/30 Write block 4 len 12288 data: 11 13 15 12/30 Write block 553 len 12288 data: 17 19 19 13/30 Read block 497 len 12288 14/30 Read block 357 len 12288 15/30 Write block 36 len 40960 data: 13 12 13 11 8 10 12 11 12 13 16/30 Read block 306 len 32768 17/30 Write block 175 len 36864 data: 17 18 15 13 14 12 10 11 9 18/30 Read block 224 len 28672 19/30 Write block 159 len 20480 data: 17 14 13 13 13 20/30 Write block 588 len 36864 data: 9 6 11 12 11 13 11 10 8 21/30 Write block 223 len 20480 data: 16 15 15 17 14 22/30 Write block 57 len 20480 data: 13 14 13 17 18 23/30 Write block 138 len 40960 data: 16 15 17 16 15 15 14 12 13 18 24/30 Flush 25/30 Write block 560 len 36864 data: 19 20 20 22 20 19 20 20 17 26/30 Read block 213 len 12288 27/30 Write block 549 len 4096 data: 16 28/30 Read block 479 len 12288 29/30 Read block 164 len 12288 30/30 Read block 592 len 32768 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 9 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 19 20 Done 22 21 3 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-17T14:35:45.889106794Z","hostname":"w-01KKY24DSS0N5K304GQQTDQ65P","pid":2090} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 195 324 324 319 F F T <--- 1 020-039 198 198 196 324 324 320 F F T <--- 2 040-059 198 198 196 324 324 320 F F F <--- 3 060-079 198 198 196 324 324 321 F F F <--- 4 080-099 197 197 197 322 322 322 F F F 5 100-119 197 197 197 322 322 322 F F F 6 120-139 198 198 190 324 324 312 F F F <--- 7 140-159 198 198 192 324 324 314 F F F <--- 8 160-179 198 198 198 324 324 323 F F F <--- 9 180-199 198 198 198 324 324 323 F F F <--- 10 200-219 192 192 192 314 314 314 F F F 11 220-239 198 198 188 324 324 308 F F F <--- 12 240-259 195 195 195 319 319 319 F F F 13 260-279 198 198 198 323 323 323 F F F 14 280-299 197 197 197 322 322 322 F F F 15 300-319 197 197 197 322 322 322 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 197 197 197 322 322 322 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 195 195 195 319 319 319 F F F 22 440-459 195 195 195 319 319 319 F F F 23 460-479 196 196 196 320 320 320 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 196 196 196 321 321 321 F F F 27 540-559 198 198 198 324 324 323 T T F <--- 28 560-579 194 194 194 316 316 316 T T F <--- 29 580-599 198 198 190 324 324 310 F F F <--- Max gen: 198, Max flush: 324 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 324 324 324 F F F 1 020-039 198 198 198 324 324 324 F F F 2 040-059 198 198 198 324 324 324 F F F 3 060-079 198 198 198 324 324 324 F F F 4 080-099 197 197 197 322 322 322 F F F 5 100-119 197 197 197 322 322 322 F F F 6 120-139 198 198 198 324 324 324 F F F 7 140-159 198 198 198 324 324 324 F F F 8 160-179 198 198 198 324 324 324 F F F 9 180-199 198 198 198 324 324 324 F F F 10 200-219 192 192 192 314 314 314 F F F 11 220-239 198 198 198 324 324 324 F F F 12 240-259 195 195 195 319 319 319 F F F 13 260-279 198 198 198 323 323 323 F F F 14 280-299 197 197 197 322 322 322 F F F 15 300-319 197 197 197 322 322 322 F F F 16 320-339 189 189 189 309 309 309 F F F 17 340-359 192 192 192 314 314 314 F F F 18 360-379 197 197 197 322 322 322 F F F 19 380-399 191 191 191 313 313 313 F F F 20 400-419 188 188 188 307 307 307 F F F 21 420-439 195 195 195 319 319 319 F F F 22 440-459 195 195 195 319 319 319 F F F 23 460-479 196 196 196 320 320 320 F F F 24 480-499 184 184 184 298 298 298 F F F 25 500-519 186 186 186 302 302 302 F F F 26 520-539 196 196 196 321 321 321 F F F 27 540-559 199 199 199 325 325 325 F F F 28 560-579 199 199 199 325 325 325 F F F 29 580-599 198 198 198 324 324 324 F F F Max gen: 199, Max flush: 325 24:07 Test duration Test completed real 24:06.722713384 user 36.092346499 sys 17.614914970 trap 0.016186373 tflt 0.074906753 dflt 0.067879640 kflt 0.003569854 lock 33:28:41.585571291 slp 1:57:50.320481552 lat 1.340569837 stop 0.024857368