Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.20951273 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.013360933 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.013011069 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.875327734Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.892234908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.894311032Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.894408773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.894569075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.894611375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Crucible c274a5df-fe76-4679-bd8e-c2ad5c64cfb6 has session id: 82c9e8ea-9453-4a5e-b009-87ccf3ae879e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.895595316Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Upstairs opts: Upstairs UUID: c274a5df-fe76-4679-bd8e-c2ad5c64cfb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.895818389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Crucible stats registered with UUID: c274a5df-fe76-4679-bd8e-c2ad5c64cfb6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.8958825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:20.896016091Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.896903761Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897128704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897143764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897328016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"c274a5df-fe76-4679-bd8e-c2ad5c64cfb6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897460428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897470188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897546059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897556749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897573299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897583739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897598549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897608809Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64463)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897786971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48187)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.897812982Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43802)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.898030404Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.899798364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.899832535Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.899859795Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900672964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900723815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900823196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900834586Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900846016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900856577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900867027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900878277Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900888837Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900899497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900910777Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900920527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900930727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.900940487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901049779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"downstairs","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901060599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901069769Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901080009Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901089989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"client":"2","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901100269Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901109709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"c274a5df-fe76-4679-bd8e-c2ad5c64cfb6 is now active with session: 82c9e8ea-9453-4a5e-b009-87ccf3ae879e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.90111992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.90113039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.90118187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"Activated sub_volume c274a5df-fe76-4679-bd8e-c2ad5c64cfb6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901246921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901288011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.901450153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY ,"time":" 1004 Acked Read 882026-07-10T05:20:20.941554272Z" Done Done Done false ,"hostname" 1005 Acked Read 256:" Done Done Done false w-01KX57DQGFNJ07NR5QMY2FD5YS" 1006 Acked Read 256 Done Done Done, false" pid":1139 STATES DS:0 DS:1 DS:2 TOTAL ," Sent task" 0 : 0 " 0 0 crutest" Done 3 3 3 9 } Skipped 0 0 0 0 Error 0 0 0 0 {Last Flush: "Some(JobId(1003)) msgSome(JobId(1003)) ":Some(JobId(1003)) " CLIENT: Tests done. All submitted work has been ACK'd"Downstairs last five completed:,"v":0,"name":"crucible","level": 100330 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 ,"time":"2026-07-10T05:20:20.941634133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:20.941737074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.943386597Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.943423468Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.943435368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.943537199Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.943552419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.943563409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.943574129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.94358575Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.94359792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.94360914Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.9436204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"82c9e8ea-9453-4a5e-b009-87ccf3ae879e"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.969152192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.976635908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.976711969Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984132124Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984391067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984414477Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984423307Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984435297Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Crucible 2df241e4-279f-416f-8213-2dc4eeb9c91c has session id: 6e554fe2-47ec-4256-90f8-79364c757c9d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984784151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Upstairs opts: Upstairs UUID: 2df241e4-279f-416f-8213-2dc4eeb9c91c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984813582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Crucible stats registered with UUID: 2df241e4-279f-416f-8213-2dc4eeb9c91c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.984824942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.984835542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985024434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985040544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985050984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985258437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985275977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"2df241e4-279f-416f-8213-2dc4eeb9c91c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985287787Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985296997Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985307237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985317327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985353078Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985364118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985374528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56288)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985412448Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33575)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985423949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41272)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.985443999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986168487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986248528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986315679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986507781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986523871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986617172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986628942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986640422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986651163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986661553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986672843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986683383Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986693973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986705183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986714943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986724973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986734534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986744304Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"":"downstairs","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986754694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986763884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"0","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986775144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"1","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986785494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"client":"2","":"downstairs","session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986795954Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986805524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"2df241e4-279f-416f-8213-2dc4eeb9c91c is now active with session: 6e554fe2-47ec-4256-90f8-79364c757c9d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986815784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986824875Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"session_id":"6e554fe2-47ec-4256-90f8-79364c757c9d"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986834355Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Activated sub_volume 2df241e4-279f-416f-8213-2dc4eeb9c91c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986842785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.986851415Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.987695625Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} 01/30 Write block 364 len 8192 data: 2 2 02/30 Read block 198 len 40960 03/30 Write block 524 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Write block 546 len 16384 data: 2 2 2 2 05/30 Read block 296 len 24576 06/30 Read block 30 len 32768 07/30 Write block 127 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Read block 223 len 32768 09/30 Flush 10/30 Write block 100 len 16384 data: 2 2 2 2 11/30 Flush 12/30 Write block 582 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Flush 14/30 Write block 6 len 4096 data: 2 15/30 Write block 415 len 12288 data: 2 2 2 16/30 Write block 275 len 24576 data: 2 2 2 2 2 2 17/30 Read block 169 len 32768 18/30 Write block 75 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Flush 20/30 Write block 329 len 24576 data: 2 2 2 2 2 2 21/30 Read block 305 len 36864 22/30 Write block 101 len 28672 data: 3 3 3 2 2 2 2 23/30 Read block 539 len 4096 24/30 Read block 498 len 20480 25/30 Flush 26/30 Flush 27/30 Write block 536 len 24576 data: 2 2 2 2 2 2 28/30 Write block 562 len 28672 data: 2 2 2 2 2 2 2 29/30 Write block 504 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Read block 89 len 16384 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Sent Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:24.997502967Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:24.99777726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 2 2 1 5 5 F F F <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 2 2 1 5 5 F F F <--- 4 080-099 1 2 2 1 5 5 F F F <--- 5 100-119 1 2 2 1 6 6 F F F <--- 6 120-139 1 2 2 1 2 2 F F F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 2 2 1 5 5 F F F <--- 14 280-299 1 2 2 1 5 5 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 2 2 1 6 6 F F F <--- 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 2 2 1 2 2 F F F <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 2 2 1 5 5 F F F <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F T T <--- 26 520-539 1 2 2 1 2 2 F T T <--- 27 540-559 1 2 2 1 2 2 F T T <--- 28 560-579 1 1 1 1 1 1 F T T <--- 29 580-599 1 2 2 1 4 4 F F F <--- Max gen: 2, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 2 2 2 5 5 5 F F F 5 100-119 2 2 2 6 6 6 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 5 5 5 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 6 6 6 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 2 2 2 5 5 5 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 3 3 3 7 7 7 F F F 26 520-539 3 3 3 7 7 7 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 3 3 3 7 7 7 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 3, Max flush: 7 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.374537737Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:29.382096713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.382168914Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389253635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389502508Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389530038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389541018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389552839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Crucible 4830b41e-3396-42d8-96c9-e9c7385db5ea has session id: 4425889c-2f3e-4770-9377-3dbd98a6db45","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389857872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Upstairs opts: Upstairs UUID: 4830b41e-3396-42d8-96c9-e9c7385db5ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389887162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Crucible stats registered with UUID: 4830b41e-3396-42d8-96c9-e9c7385db5ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.389898533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:29.389914293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390105755Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390137185Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390148655Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390237596Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390256757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"4830b41e-3396-42d8-96c9-e9c7385db5ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390275077Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390284277Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390294227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390304127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390314237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390326677Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390336928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"ds_connection connected from Ok(127.0.0.1:40102)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390439099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"ds_connection connected from Ok(127.0.0.1:63305)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.390469889Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"ds_connection connected from Ok(127.0.0.1:36472)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.39051735Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391082446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391102276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391300239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.39142372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[0]R flush_numbers[0..12]: [5, 1, 1, 5, 5, 6, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391616912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[0]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391628312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391639762Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[1]R flush_numbers[0..12]: [5, 1, 1, 5, 5, 6, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391650453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[1]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391663323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391674693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[2]R flush_numbers[0..12]: [5, 1, 1, 5, 5, 6, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391685303Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[2]R generation[0..12]: [2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391695803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391707103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391716883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391726863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391736644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391746324Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"":"downstairs","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391756344Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391765494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"0","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391775534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"1","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391785554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"client":"2","":"downstairs","session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391795764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391805244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"4830b41e-3396-42d8-96c9-e9c7385db5ea is now active with session: 4425889c-2f3e-4770-9377-3dbd98a6db45","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391815434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391825845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"session_id":"4425889c-2f3e-4770-9377-3dbd98a6db45"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391835475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Activated sub_volume 4830b41e-3396-42d8-96c9-e9c7385db5ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.391843895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-07-10T05:20:29.391852345Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.392201839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} 01/30 Write block 105 len 28672 data: 3 3 3 2 2 2 2 02/30 Read block 172 len 8192 03/30 Flush 04/30 Write block 410 len 16384 data: 2 2 2 2 05/30 Read block 68 len 36864 06/30 Write block 255 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 430 len 16384 08/30 Write block 489 len 4096 data: 2 09/30 Read block 280 len 8192 10/30 Read block 297 len 4096 11/30 Read block 74 len 40960 12/30 Write block 177 len 24576 data: 2 2 2 2 2 2 13/30 Write block 258 len 40960 data: 3 3 3 3 2 2 2 2 2 2 14/30 Write block 346 len 8192 data: 2 2 15/30 Read block 236 len 36864 16/30 Flush 17/30 Write block 68 len 20480 data: 2 2 2 2 2 18/30 Read block 514 len 4096 19/30 Write block 382 len 16384 data: 2 2 2 2 20/30 Read block 46 len 4096 21/30 Write block 185 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Read block 267 len 40960 23/30 Write block 332 len 8192 data: 3 3 24/30 Write block 7 len 8192 data: 2 2 25/30 Write block 439 len 16384 data: 2 2 2 2 26/30 Read block 436 len 28672 27/30 Read block 211 len 28672 28/30 Read block 72 len 20480 29/30 Write block 84 len 24576 data: 2 2 2 2 2 2 30/30 Write block 11 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:29.400244951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:29.400526154Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 5 5 5 F T T <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 5 5 5 F T T <--- 4 080-099 2 2 2 5 5 5 F T T <--- 5 100-119 2 4 4 6 8 8 T F F <--- 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 4 4 1 9 9 F F F <--- 9 180-199 1 4 4 1 9 9 F T T <--- 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 4 4 1 9 9 F F F <--- 13 260-279 2 4 4 5 9 9 F F F <--- 14 280-299 2 2 2 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 6 6 6 F T T <--- 17 340-359 1 4 4 1 9 9 F F F <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 1 1 1 1 1 1 F T T <--- 20 400-419 2 4 4 5 9 9 F F F <--- 21 420-439 1 1 1 1 1 1 F T T <--- 22 440-459 1 1 1 1 1 1 F T T <--- 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 4 4 1 9 9 F F F <--- 25 500-519 3 3 3 7 7 7 F F F 26 520-539 3 3 3 7 7 7 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 3 3 3 7 7 7 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 4, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 10 10 10 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 5 5 5 10 10 10 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 5 5 5 10 10 10 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 4 4 4 9 9 9 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 5 5 5 10 10 10 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 3 3 3 7 7 7 F F F 26 520-539 3 3 3 7 7 7 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 3 3 3 7 7 7 F F F 29 580-599 2 2 2 4 4 4 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.65665678Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.665439811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.665745614Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.665780715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.665790335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.665802285Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Crucible c0bfd34d-c57a-4d94-a71b-44eaa5685781 has session id: 5578c6c5-f178-4f96-8cf7-373b9efacb7e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666128629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Upstairs opts: Upstairs UUID: c0bfd34d-c57a-4d94-a71b-44eaa5685781, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666177989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Crucible stats registered with UUID: c0bfd34d-c57a-4d94-a71b-44eaa5685781","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666189079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:33.666198819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666377001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666392552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666402952Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666522723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666538553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"c0bfd34d-c57a-4d94-a71b-44eaa5685781 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666572754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666582614Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666592824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666602854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666613014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666623554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666634094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50633)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666679675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37821)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666706045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64566)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.666717775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667313672Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667342862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667433493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667632186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667649596Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[0]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 8, 2, 1, 9, 10, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667707507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[0]R generation[0..12]: [5, 1, 1, 5, 5, 4, 2, 1, 4, 5, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667718887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667730397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[1]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 8, 2, 1, 9, 10, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667741007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[1]R generation[0..12]: [5, 1, 1, 5, 5, 4, 2, 1, 4, 5, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667751427Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667762767Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[2]R flush_numbers[0..12]: [10, 1, 1, 10, 10, 8, 2, 1, 9, 10, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667773467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[2]R generation[0..12]: [5, 1, 1, 5, 5, 4, 2, 1, 4, 5, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667783827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667794938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667804828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667816328Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667825968Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667835898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"":"downstairs","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667846188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667855408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"0","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667865758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"1","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667875999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"client":"2","":"downstairs","session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667886439Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667896229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"c0bfd34d-c57a-4d94-a71b-44eaa5685781 is now active with session: 5578c6c5-f178-4f96-8cf7-373b9efacb7e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667906709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667915869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"session_id":"5578c6c5-f178-4f96-8cf7-373b9efacb7e"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667925269Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Activated sub_volume c0bfd34d-c57a-4d94-a71b-44eaa5685781","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667933599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.667942009Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.668346074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} 01/30 Flush 02/30 Write block 316 len 16384 data: 2 2 2 2 03/30 Write block 194 len 20480 data: 3 2 2 2 2 04/30 Read block 435 len 4096 05/30 Write block 316 len 32768 data: 3 3 3 3 2 2 2 2 06/30 Read block 179 len 36864 07/30 Write block 533 len 24576 data: 2 2 2 3 3 3 08/30 Write block 310 len 28672 data: 2 2 2 2 2 2 4 09/30 Read block 532 len 20480 10/30 Read block 274 len 40960 11/30 Read block 323 len 16384 12/30 Read block 149 len 28672 13/30 Write block 37 len 4096 data: 2 14/30 Read block 555 len 24576 15/30 Write block 584 len 40960 data: 3 3 3 3 3 3 2 2 2 2 16/30 Write block 453 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Flush 18/30 Write block 9 len 24576 data: 2 2 3 3 3 3 19/30 Write block 180 len 40960 data: 3 3 3 2 2 3 3 3 3 3 20/30 Read block 168 len 40960 21/30 Write block 76 len 8192 data: 3 3 22/30 Read block 128 len 24576 23/30 Read block 25 len 28672 24/30 Read block 167 len 40960 25/30 Write block 575 len 40960 data: 2 2 2 2 2 2 2 3 3 4 26/30 Read block 50 len 36864 27/30 Write block 3 len 36864 data: 2 2 2 3 3 3 3 3 4 28/30 Write block 435 len 16384 data: 2 2 2 2 29/30 Write block 500 len 28672 data: 2 2 2 2 3 3 3 30/30 Read block 503 len 4096 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 28 0 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1000)) Some(JobId(1016)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:33.677326287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:33.6776014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 10 10 10 T F T <--- 1 020-039 6 1 6 12 1 12 F F F <--- 2 040-059 1 1 1 1 1 1 F F F 3 060-079 5 5 5 10 10 10 T F T <--- 4 080-099 5 5 5 10 10 10 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 6 5 6 12 10 12 T F T <--- 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 6 1 6 12 1 12 F F F <--- 16 320-339 6 5 6 12 10 12 F F F <--- 17 340-359 4 4 4 9 9 9 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 5 5 5 10 10 10 T F T <--- 22 440-459 6 5 6 12 10 12 F F F <--- 23 460-479 6 1 6 12 1 12 F F F <--- 24 480-499 4 4 4 9 9 9 F F F 25 500-519 3 3 3 7 7 7 T F T <--- 26 520-539 6 3 6 12 7 12 F F F <--- 27 540-559 3 3 3 7 7 7 F F F 28 560-579 3 3 3 7 7 7 T F T <--- 29 580-599 6 2 6 12 4 12 T F T <--- Max gen: 6, Max flush: 12 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 7 7 7 13 13 13 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 6 6 6 12 12 12 F F F 17 340-359 4 4 4 9 9 9 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 7 7 7 13 13 13 F F F 22 440-459 6 6 6 12 12 12 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 7 7 7 13 13 13 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 7 7 7 13 13 13 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 7, Max flush: 13 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.920284391Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:37.927779377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.927888888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.935360323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.935620786Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.935648567Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.935657447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.935669337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Crucible 33438fd1-553f-4997-a71e-b27794a2611f has session id: fd6147af-9550-4bfb-8795-e7da807c3580","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936057861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Upstairs opts: Upstairs UUID: 33438fd1-553f-4997-a71e-b27794a2611f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936105362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Crucible stats registered with UUID: 33438fd1-553f-4997-a71e-b27794a2611f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936117622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:37.936127302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936308104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936338895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936350385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936479446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936496856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"33438fd1-553f-4997-a71e-b27794a2611f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936513377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936522977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936532997Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936543037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936553067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936563537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936573657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"ds_connection connected from Ok(127.0.0.1:44500)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936626948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"ds_connection connected from Ok(127.0.0.1:51390)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936643468Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"ds_connection connected from Ok(127.0.0.1:48667)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.936654528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937396597Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937439227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937525198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937729711Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937784191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[0]R flush_numbers[0..12]: [13, 12, 1, 13, 10, 8, 2, 1, 9, 13, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937809762Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[0]R generation[0..12]: [7, 6, 1, 7, 5, 4, 2, 1, 4, 7, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937821202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937832682Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[1]R flush_numbers[0..12]: [13, 12, 1, 13, 10, 8, 2, 1, 9, 13, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937843462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[1]R generation[0..12]: [7, 6, 1, 7, 5, 4, 2, 1, 4, 7, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937854032Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937865442Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[2]R flush_numbers[0..12]: [13, 12, 1, 13, 10, 8, 2, 1, 9, 13, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937876222Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[2]R generation[0..12]: [7, 6, 1, 7, 5, 4, 2, 1, 4, 7, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937886622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937897903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937907593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937917723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937927413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937942873Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"":"downstairs","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937952993Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937962193Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"0","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937973313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"1","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937983313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"client":"2","":"downstairs","session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.937993584Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938015084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"33438fd1-553f-4997-a71e-b27794a2611f is now active with session: fd6147af-9550-4bfb-8795-e7da807c3580","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938025384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938034704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"session_id":"fd6147af-9550-4bfb-8795-e7da807c3580"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938044084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Activated sub_volume 33438fd1-553f-4997-a71e-b27794a2611f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938052644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938061164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.938488439Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} 01/30 Write block 250 len 40960 data: 2 2 2 2 2 3 3 3 4 4 02/30 Read block 130 len 8192 03/30 Read block 427 len 28672 04/30 Flush 05/30 Write block 341 len 8192 data: 2 2 06/30 Read block 105 len 16384 07/30 Read block 305 len 28672 08/30 Write block 404 len 16384 data: 2 2 2 2 09/30 Read block 339 len 40960 10/30 Read block 10 len 32768 11/30 Read block 485 len 40960 12/30 Write block 238 len 20480 data: 2 2 2 2 2 13/30 Read block 2 len 16384 14/30 Read block 400 len 32768 15/30 Write block 332 len 40960 data: 4 4 3 2 2 2 2 2 2 3 16/30 Read block 112 len 8192 17/30 Write block 90 len 16384 data: 2 2 2 2 18/30 Read block 151 len 4096 19/30 Write block 523 len 20480 data: 2 3 3 3 3 20/30 Read block 518 len 24576 21/30 Write block 306 len 32768 data: 2 2 2 2 3 3 3 3 22/30 Read block 532 len 16384 23/30 Read block 114 len 32768 24/30 Write block 391 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 191 len 20480 data: 3 3 3 4 3 26/30 Write block 450 len 8192 data: 2 2 27/30 Write block 206 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 434 len 4096 29/30 Read block 514 len 40960 30/30 Read block 44 len 4096 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 1 28 Done 3 30 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:37.947362541Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:37.947652114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 5 5 5 10 10 10 F T T <--- 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 7 7 7 13 13 13 F T T <--- 10 200-219 1 1 1 1 1 1 F T T <--- 11 220-239 1 1 1 1 1 1 F T T <--- 12 240-259 4 8 8 9 14 14 T T T <--- 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 6 6 6 12 12 12 F T T <--- 16 320-339 6 6 6 12 12 12 F T T <--- 17 340-359 4 4 4 9 9 9 F T T <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 5 5 5 10 10 10 F T T <--- 20 400-419 4 4 4 9 9 9 F T T <--- 21 420-439 7 7 7 13 13 13 F F F 22 440-459 6 6 6 12 12 12 F T T <--- 23 460-479 6 6 6 12 12 12 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 7 7 7 13 13 13 F F F 26 520-539 6 6 6 12 12 12 F T T <--- 27 540-559 3 3 3 7 7 7 F F F 28 560-579 7 7 7 13 13 13 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 8, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 9 9 9 15 15 15 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 9 9 9 15 15 15 F F F 13 260-279 4 4 4 9 9 9 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 9 9 9 15 15 15 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 9 9 9 15 15 15 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 9 9 9 15 15 15 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 7 7 7 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 7 7 7 13 13 13 F F F 26 520-539 9 9 9 15 15 15 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 7 7 7 13 13 13 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.19500595Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:42.202425805Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.202501325Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.209494995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.209756028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.209779609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.209790529Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.209802619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Crucible 300559d7-e6b5-4c0a-8d46-d306323ea679 has session id: 8bd39012-1f57-4ced-8f7b-15d1f558737f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210158513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Upstairs opts: Upstairs UUID: 300559d7-e6b5-4c0a-8d46-d306323ea679, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210190053Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Crucible stats registered with UUID: 300559d7-e6b5-4c0a-8d46-d306323ea679","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210201714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:42.210212694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210395326Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210411036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210421636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210564028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210589548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"300559d7-e6b5-4c0a-8d46-d306323ea679 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210606668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210616018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210626108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210636259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210646309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210658709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210669309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62044)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210717049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37206)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.21080555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53240)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.210824201Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211336607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211362367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211461838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211704281Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211714811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[0]R flush_numbers[0..12]: [13, 12, 1, 13, 15, 8, 2, 1, 9, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211777712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[0]R generation[0..12]: [7, 6, 1, 7, 9, 4, 2, 1, 4, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211789112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211800752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[1]R flush_numbers[0..12]: [13, 12, 1, 13, 15, 8, 2, 1, 9, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211811632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[1]R generation[0..12]: [7, 6, 1, 7, 9, 4, 2, 1, 4, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211824842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211836312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[2]R flush_numbers[0..12]: [13, 12, 1, 13, 15, 8, 2, 1, 9, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211847042Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[2]R generation[0..12]: [7, 6, 1, 7, 9, 4, 2, 1, 4, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211857523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211868583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211878493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211888693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211898533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211908293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"":"downstairs","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211918633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211927703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"0","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211937763Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"1","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211947964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"client":"2","":"downstairs","session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211958234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211967674Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"300559d7-e6b5-4c0a-8d46-d306323ea679 is now active with session: 8bd39012-1f57-4ced-8f7b-15d1f558737f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211977994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211988204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"session_id":"8bd39012-1f57-4ced-8f7b-15d1f558737f"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.211997904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Activated sub_volume 300559d7-e6b5-4c0a-8d46-d306323ea679","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.212006904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.212015574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.212464399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} 01/30 Flush 02/30 Write block 323 len 4096 data: 3 03/30 Write block 256 len 12288 data: 4 4 5 04/30 Flush 05/30 Write block 528 len 24576 data: 3 3 3 3 3 3 06/30 Read block 89 len 4096 07/30 Read block 57 len 16384 08/30 Read block 566 len 20480 09/30 Read block 176 len 28672 10/30 Write block 48 len 16384 data: 2 2 2 2 11/30 Read block 229 len 20480 12/30 Write block 349 len 12288 data: 2 2 2 13/30 Read block 157 len 24576 14/30 Write block 90 len 40960 data: 3 3 3 3 2 2 2 2 2 2 15/30 Read block 518 len 16384 16/30 Write block 170 len 4096 data: 2 17/30 Read block 116 len 24576 18/30 Read block 237 len 24576 19/30 Read block 38 len 32768 20/30 Read block 3 len 8192 21/30 Read block 234 len 12288 22/30 Write block 573 len 16384 data: 2 2 3 3 23/30 Read block 147 len 40960 24/30 Flush 25/30 Write block 261 len 8192 data: 4 3 26/30 Write block 96 len 8192 data: 3 3 27/30 Write block 496 len 24576 data: 2 2 2 2 3 3 28/30 Write block 382 len 28672 data: 3 3 3 3 2 2 2 29/30 Read block 8 len 16384 30/30 Write block 292 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 4 Sent Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 2 25 Done 4 25 24 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:42.22041062Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:42.220727014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 1 10 10 1 18 18 F F F <--- 3 060-079 7 7 7 13 13 13 F F F 4 080-099 9 10 10 15 18 18 F T T <--- 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 4 10 10 9 18 18 F F F <--- 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 4 4 4 9 9 9 F T T <--- 14 280-299 2 2 2 5 5 5 F T T <--- 15 300-319 9 9 9 15 15 15 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 9 10 10 15 18 18 F F F <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 9 9 9 15 15 15 F T T <--- 20 400-419 9 9 9 15 15 15 F F F 21 420-439 7 7 7 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 4 4 4 9 9 9 F T T <--- 25 500-519 7 7 7 13 13 13 F T T <--- 26 520-539 9 10 10 15 18 18 T F F <--- 27 540-559 3 3 3 7 7 7 F F F 28 560-579 7 10 10 13 18 18 F F F <--- 29 580-599 7 7 7 13 13 13 F F F Max gen: 10, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 10 10 10 18 18 18 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 11 11 11 19 19 19 F F F 5 100-119 4 4 4 8 8 8 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 10 10 10 17 17 17 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 9 9 9 15 15 15 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 7 7 7 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 6 6 6 12 12 12 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 11 11 11 19 19 19 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 3 3 3 7 7 7 F F F 28 560-579 10 10 10 18 18 18 F F F 29 580-599 7 7 7 13 13 13 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.467092738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.475889059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476198262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476222083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476231343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476243363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Crucible 19291f1b-667a-4610-a790-747b3c1c208e has session id: b8c295d1-43fa-4248-a43d-ad3afc2e11d5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476597377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Upstairs opts: Upstairs UUID: 19291f1b-667a-4610-a790-747b3c1c208e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476645848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Crucible stats registered with UUID: 19291f1b-667a-4610-a790-747b3c1c208e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.476656868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:46.476666388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.47684612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.47686467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.47687536Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477031602Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477048532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"19291f1b-667a-4610-a790-747b3c1c208e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477080513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477090273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477100403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477110443Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477120583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477131133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477141363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39270)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477167144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61390)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477178464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:55905)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477197834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477884202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477943102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.477961363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478219056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478248446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[0]R flush_numbers[0..12]: [13, 12, 18, 13, 19, 8, 2, 1, 18, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478300616Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[0]R generation[0..12]: [7, 6, 10, 7, 11, 4, 2, 1, 10, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478312087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478323657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[1]R flush_numbers[0..12]: [13, 12, 18, 13, 19, 8, 2, 1, 18, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478334607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[1]R generation[0..12]: [7, 6, 10, 7, 11, 4, 2, 1, 10, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478345357Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478356817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[2]R flush_numbers[0..12]: [13, 12, 18, 13, 19, 8, 2, 1, 18, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478367557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[2]R generation[0..12]: [7, 6, 10, 7, 11, 4, 2, 1, 10, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478378377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478389927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478399898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478410198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478421498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478431468Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"":"downstairs","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478441788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478451108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"0","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478461438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"1","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478471538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"client":"2","":"downstairs","session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478481869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478491549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"19291f1b-667a-4610-a790-747b3c1c208e is now active with session: b8c295d1-43fa-4248-a43d-ad3afc2e11d5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478501989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478511229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"session_id":"b8c295d1-43fa-4248-a43d-ad3afc2e11d5"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478520859Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Activated sub_volume 19291f1b-667a-4610-a790-747b3c1c208e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478529359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478538019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.478947604Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} 01/30 Write block 470 len 20480 data: 2 2 2 2 2 02/30 Read block 226 len 24576 03/30 Write block 430 len 20480 data: 2 2 2 2 2 04/30 Write block 104 len 40960 data: 3 4 4 4 3 3 3 3 2 2 05/30 Write block 113 len 4096 data: 3 06/30 Read block 565 len 16384 07/30 Write block 558 len 8192 data: 2 2 08/30 Read block 562 len 24576 09/30 Read block 50 len 4096 10/30 Flush 11/30 Write block 45 len 4096 data: 2 12/30 Read block 549 len 32768 13/30 Write block 541 len 28672 data: 3 2 2 2 2 3 3 14/30 Write block 256 len 4096 data: 5 15/30 Write block 6 len 28672 data: 4 4 4 4 4 5 4 16/30 Write block 303 len 12288 data: 2 2 2 17/30 Read block 591 len 16384 18/30 Write block 92 len 28672 data: 4 4 3 3 4 4 3 19/30 Read block 328 len 40960 20/30 Write block 432 len 32768 data: 3 3 3 3 3 3 3 3 21/30 Read block 214 len 28672 22/30 Write block 239 len 4096 data: 3 23/30 Write block 545 len 12288 data: 3 4 4 24/30 Write block 80 len 12288 data: 3 3 3 25/30 Read block 458 len 28672 26/30 Write block 573 len 40960 data: 3 3 4 4 3 3 3 3 3 4 27/30 Read block 509 len 4096 28/30 Write block 507 len 36864 data: 3 3 3 3 3 2 2 2 2 29/30 Read block 457 len 12288 30/30 Read block 344 len 24576 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Read 1 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 22 22 Done 30 30 8 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:46.487096647Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:46.48737636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 T T F <--- 1 020-039 6 6 6 12 12 12 F F F 2 040-059 10 10 10 18 18 18 T T F <--- 3 060-079 7 7 7 13 13 13 F F F 4 080-099 11 11 11 19 19 19 T T F <--- 5 100-119 12 12 4 20 20 8 F F T <--- 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 T T F <--- 12 240-259 10 10 10 17 17 17 T T F <--- 13 260-279 11 11 11 19 19 19 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 9 9 9 15 15 15 T T F <--- 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 12 12 7 20 20 13 T T T <--- 22 440-459 9 9 9 15 15 15 F F F 23 460-479 12 12 6 20 20 12 F F T <--- 24 480-499 11 11 11 19 19 19 F F F 25 500-519 11 11 11 19 19 19 T T F <--- 26 520-539 10 10 10 18 18 18 F F F 27 540-559 12 12 3 20 20 7 T T T <--- 28 560-579 10 10 10 18 18 18 T T F <--- 29 580-599 7 7 7 13 13 13 T T F <--- Max gen: 12, Max flush: 20 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 7 7 7 13 13 13 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 12 12 12 20 20 20 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 13 13 13 21 21 21 F F F 12 240-259 13 13 13 21 21 21 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 13 13 13 21 21 21 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 12 12 12 20 20 20 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 13 13 13 21 21 21 F F F 28 560-579 13 13 13 21 21 21 F F F 29 580-599 13 13 13 21 21 21 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.731006003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.739612642Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.739904405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.739928475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.739940715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.739952976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Crucible 0adde5be-622f-4dc7-85f8-50fdbb87ecf6 has session id: 49315c65-c88a-4c2c-8715-8649d27a2c75","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74034866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Upstairs opts: Upstairs UUID: 0adde5be-622f-4dc7-85f8-50fdbb87ecf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740390051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Crucible stats registered with UUID: 0adde5be-622f-4dc7-85f8-50fdbb87ecf6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740401221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:50.740411061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740582613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740598803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740609313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740741935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740756915Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"0adde5be-622f-4dc7-85f8-50fdbb87ecf6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740780555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740790355Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740800375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740811935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740822086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740832646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740843076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:44777)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740902606Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:61281)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740928937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"ds_connection connected from Ok(127.0.0.1:47690)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.740940717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.741629085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.741691265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.741704466Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.741965559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742005559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[0]R flush_numbers[0..12]: [21, 12, 21, 13, 21, 20, 2, 1, 18, 15, 15, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74205594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[0]R generation[0..12]: [13, 6, 13, 7, 13, 12, 2, 1, 10, 9, 9, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74206774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74207963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[1]R flush_numbers[0..12]: [21, 12, 21, 13, 21, 20, 2, 1, 18, 15, 15, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74209231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[1]R generation[0..12]: [13, 6, 13, 7, 13, 12, 2, 1, 10, 9, 9, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74210336Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74211523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[2]R flush_numbers[0..12]: [21, 12, 21, 13, 21, 20, 2, 1, 18, 15, 15, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.74212647Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[2]R generation[0..12]: [13, 6, 13, 7, 13, 12, 2, 1, 10, 9, 9, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742137381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742148961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742159021Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742169351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742179361Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742189401Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"":"downstairs","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742199751Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742209051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"0","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742219472Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"1","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742229712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"client":"2","":"downstairs","session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742240152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742251022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"0adde5be-622f-4dc7-85f8-50fdbb87ecf6 is now active with session: 49315c65-c88a-4c2c-8715-8649d27a2c75","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742261702Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742270982Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"session_id":"49315c65-c88a-4c2c-8715-8649d27a2c75"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742280662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Activated sub_volume 0adde5be-622f-4dc7-85f8-50fdbb87ecf6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742289552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742298202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.742730137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} 01/30 Write block 232 len 32768 data: 2 2 2 2 2 2 3 4 02/30 Write block 572 len 36864 data: 2 4 4 5 5 4 4 4 4 03/30 Read block 153 len 12288 04/30 Write block 541 len 40960 data: 4 3 3 3 4 5 5 3 3 2 05/30 Read block 252 len 40960 06/30 Flush 07/30 Write block 127 len 32768 data: 3 3 3 3 3 3 3 3 08/30 Flush 09/30 Write block 133 len 24576 data: 4 4 2 2 2 2 10/30 Read block 356 len 20480 11/30 Flush 12/30 Write block 59 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Read block 96 len 20480 14/30 Read block 442 len 24576 15/30 Write block 259 len 28672 data: 5 4 5 4 3 3 3 16/30 Flush 17/30 Write block 105 len 40960 data: 5 5 5 4 4 4 4 3 4 2 18/30 Write block 192 len 8192 data: 4 4 19/30 Write block 161 len 36864 data: 2 2 2 2 2 2 2 2 2 20/30 Write block 73 len 24576 data: 2 2 3 4 4 3 21/30 Read block 29 len 8192 22/30 Write block 173 len 32768 data: 2 2 2 2 3 3 3 4 23/30 Read block 405 len 32768 24/30 Write block 469 len 40960 data: 2 3 3 3 3 3 2 2 2 2 25/30 Flush 26/30 Write block 55 len 12288 data: 2 2 2 27/30 Read block 505 len 24576 28/30 Write block 106 len 16384 data: 6 6 5 5 29/30 Read block 275 len 4096 30/30 Write block 463 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 23 25 Done 23 23 1 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:50.752074464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:50.752353378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 14 14 13 25 25 21 T T F <--- 3 060-079 14 14 7 26 26 13 F F F <--- 4 080-099 13 13 13 21 21 21 F F F 5 100-119 14 14 12 26 26 20 T T F <--- 6 120-139 14 14 2 24 24 2 F F T <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 10 26 26 18 F F F <--- 9 180-199 14 14 9 26 26 15 F F F <--- 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 14 14 13 25 25 21 F F F <--- 13 260-279 14 14 11 25 25 19 F F F <--- 14 280-299 11 11 11 19 19 19 F F F 15 300-319 13 13 13 21 21 21 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 14 14 12 26 26 20 T T F <--- 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 14 14 14 22 22 22 F F F 29 580-599 14 14 14 22 22 22 F F F Max gen: 14, Max flush: 26 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 14 14 14 25 25 25 F F F 13 260-279 14 14 14 25 25 25 F F F 14 280-299 11 11 11 19 19 19 F F F 15 300-319 13 13 13 21 21 21 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 13 13 13 21 21 21 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 14 14 14 22 22 22 F F F 29 580-599 14 14 14 22 22 22 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:54.990877181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:54.998398137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:54.998480788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006250847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.00655294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006577831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006586961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006598971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Crucible 3ebe6222-55c5-4ed7-a5a6-6fdc66355505 has session id: 4cb942c2-ec28-481f-b364-5b0f9a5484a7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006938445Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Upstairs opts: Upstairs UUID: 3ebe6222-55c5-4ed7-a5a6-6fdc66355505, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.006986335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Crucible stats registered with UUID: 3ebe6222-55c5-4ed7-a5a6-6fdc66355505","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007014646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:55.007024766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007186388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007202738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007213348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.00740162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.00742729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"3ebe6222-55c5-4ed7-a5a6-6fdc66355505 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007446231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007455691Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007465861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007475841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007485861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007496351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007506671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57269)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007619723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58464)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007667983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39328)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.007702894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.00826983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008372441Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008448162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008596414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008638854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[0]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 27, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008769206Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[0]R generation[0..12]: [13, 6, 15, 14, 13, 15, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008780796Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008792626Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[1]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 27, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008803376Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[1]R generation[0..12]: [13, 6, 15, 14, 13, 15, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008813986Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008825376Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[2]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 27, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008836287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[2]R generation[0..12]: [13, 6, 15, 14, 13, 15, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008849547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008861027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008871007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008881187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008891027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008900897Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"":"downstairs","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008910937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008920267Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"0","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008930348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"1","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008940558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"client":"2","":"downstairs","session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008950968Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008960688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"3ebe6222-55c5-4ed7-a5a6-6fdc66355505 is now active with session: 4cb942c2-ec28-481f-b364-5b0f9a5484a7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008971108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008980458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"session_id":"4cb942c2-ec28-481f-b364-5b0f9a5484a7"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.008990008Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Activated sub_volume 3ebe6222-55c5-4ed7-a5a6-6fdc66355505","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.009011169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.009020099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.009471764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} 01/30 Write block 579 len 40960 data: 5 5 4 5 4 5 4 4 4 4 02/30 Read block 350 len 32768 03/30 Write block 400 len 24576 data: 2 2 2 2 3 3 04/30 Read block 333 len 12288 05/30 Write block 568 len 36864 data: 3 2 2 2 3 5 5 6 6 06/30 Read block 405 len 8192 07/30 Write block 342 len 32768 data: 3 2 2 2 3 3 2 3 08/30 Write block 103 len 16384 data: 4 4 6 7 09/30 Read block 566 len 20480 10/30 Read block 343 len 12288 11/30 Read block 44 len 8192 12/30 Read block 43 len 40960 13/30 Read block 173 len 12288 14/30 Write block 435 len 36864 data: 4 4 4 4 4 3 3 3 2 15/30 Read block 140 len 28672 16/30 Write block 274 len 20480 data: 2 3 3 3 3 17/30 Read block 345 len 32768 18/30 Read block 164 len 28672 19/30 Flush 20/30 Write block 393 len 32768 data: 3 3 3 3 3 3 2 3 21/30 Read block 200 len 32768 22/30 Write block 551 len 20480 data: 2 2 2 2 2 23/30 Write block 558 len 36864 data: 3 3 2 2 3 3 3 3 3 24/30 Write block 404 len 4096 data: 4 25/30 Read block 439 len 20480 26/30 Read block 379 len 28672 27/30 Write block 387 len 36864 data: 3 3 2 2 3 3 4 4 4 28/30 Write block 437 len 12288 data: 5 5 5 29/30 Write block 299 len 24576 data: 2 2 2 2 3 3 30/30 Read block 359 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 0 31 Done 1 28 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:55.018923422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:55.019240926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 15 16 16 27 28 28 F F F <--- 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 14 14 14 25 25 25 F F F 13 260-279 14 16 16 25 28 28 F F F <--- 14 280-299 11 11 11 19 19 19 F T T <--- 15 300-319 13 13 13 21 21 21 F T T <--- 16 320-339 10 10 10 17 17 17 F F F 17 340-359 10 16 16 18 28 28 F F F <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 11 11 11 19 19 19 F T T <--- 20 400-419 9 16 16 15 28 28 F T T <--- 21 420-439 13 16 16 21 28 28 F T T <--- 22 440-459 9 16 16 15 28 28 F F F <--- 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 14 14 14 22 22 22 F T T <--- 28 560-579 14 16 16 22 28 28 T T T <--- 29 580-599 14 16 16 22 28 28 T F F <--- Max gen: 16, Max flush: 28 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 16 16 16 28 28 28 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 14 14 14 25 25 25 F F F 13 260-279 16 16 16 28 28 28 F F F 14 280-299 17 17 17 29 29 29 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 17 17 17 29 29 29 F F F 21 420-439 17 17 17 29 29 29 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 10 10 10 18 18 18 F F F 27 540-559 17 17 17 29 29 29 F F F 28 560-579 17 17 17 29 29 29 F F F 29 580-599 16 16 16 28 28 28 F F F Max gen: 17, Max flush: 29 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.269740457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:59.277306504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.277381765Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.284529906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.284789109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28481612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28482618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28483819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Crucible 74838271-d1cc-47b4-9bfd-78c8590524f1 has session id: 2221ccdb-636e-4a24-9938-fbf98bcb332c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285220014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Upstairs opts: Upstairs UUID: 74838271-d1cc-47b4-9bfd-78c8590524f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285252565Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Crucible stats registered with UUID: 74838271-d1cc-47b4-9bfd-78c8590524f1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285263645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:59.285273335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285442767Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285471757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285483157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285650019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28567687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"74838271-d1cc-47b4-9bfd-78c8590524f1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28568857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285698Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28570872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285719Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.28573941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.2857497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45281)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285832861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61921)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285859392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53051)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.285871142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286635801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286664011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286907554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286927754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[0]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 28, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286981704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[0]R generation[0..12]: [13, 6, 15, 14, 13, 16, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.286993265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287004985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[1]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 28, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287016105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[1]R generation[0..12]: [13, 6, 15, 14, 13, 16, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287026895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287038215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[2]R flush_numbers[0..12]: [21, 12, 27, 26, 21, 28, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287048985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[2]R generation[0..12]: [13, 6, 15, 14, 13, 16, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287059685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287070865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287080636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287090836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287101926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287111946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"":"downstairs","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287122116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287131536Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"0","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287141666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"1","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287151716Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"client":"2","":"downstairs","session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287162177Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287171777Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"74838271-d1cc-47b4-9bfd-78c8590524f1 is now active with session: 2221ccdb-636e-4a24-9938-fbf98bcb332c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287182167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287191397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"session_id":"2221ccdb-636e-4a24-9938-fbf98bcb332c"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287201077Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Activated sub_volume 74838271-d1cc-47b4-9bfd-78c8590524f1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287209747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287218317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.287609862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} 01/30 Write block 516 len 12288 data: 2 2 2 02/30 Read block 141 len 28672 03/30 Read block 109 len 40960 04/30 Read block 432 len 40960 05/30 Write block 95 len 24576 data: 4 5 5 4 3 3 06/30 Write block 427 len 28672 data: 2 2 2 3 3 4 4 07/30 Write block 578 len 8192 data: 5 6 08/30 Read block 51 len 36864 09/30 Read block 266 len 4096 10/30 Read block 157 len 40960 11/30 Read block 165 len 12288 12/30 Write block 546 len 36864 data: 6 6 4 4 3 3 3 3 3 13/30 Read block 90 len 40960 14/30 Read block 292 len 40960 15/30 Flush 16/30 Write block 51 len 24576 data: 3 2 2 2 3 3 17/30 Write block 538 len 40960 data: 4 3 3 5 4 4 4 5 7 7 18/30 Read block 133 len 12288 19/30 Write block 242 len 28672 data: 3 2 2 2 2 2 2 20/30 Read block 78 len 16384 21/30 Read block 205 len 20480 22/30 Read block 37 len 40960 23/30 Read block 197 len 12288 24/30 Read block 448 len 24576 25/30 Read block 9 len 32768 26/30 Read block 191 len 12288 27/30 Read block 538 len 4096 28/30 Write block 262 len 40960 data: 5 4 4 4 3 3 2 2 2 2 29/30 Read block 175 len 16384 30/30 Read block 222 len 32768 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:20:59.297144831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:20:59.297412654Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 15 15 15 27 27 27 F T T <--- 3 060-079 14 14 14 26 26 26 F F F 4 080-099 13 18 18 21 30 30 F F F <--- 5 100-119 16 18 18 28 30 30 F F F <--- 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 14 14 14 25 25 25 F T T <--- 13 260-279 16 16 16 28 28 28 F T T <--- 14 280-299 17 17 17 29 29 29 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 17 17 17 29 29 29 F F F 21 420-439 17 18 18 29 30 30 F F F <--- 22 440-459 16 16 16 28 28 28 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 13 18 18 21 30 30 F F F <--- 26 520-539 10 10 10 18 18 18 F T T <--- 27 540-559 17 18 18 29 30 30 F T T <--- 28 560-579 17 18 18 29 30 30 F F F <--- 29 580-599 16 16 16 28 28 28 F F F Max gen: 18, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 21 21 21 F F F 1 020-039 6 6 6 12 12 12 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 18 18 18 30 30 30 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 19 19 19 31 31 31 F F F 14 280-299 17 17 17 29 29 29 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 10 10 10 17 17 17 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 17 17 17 29 29 29 F F F 21 420-439 18 18 18 30 30 30 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 16 16 16 28 28 28 F F F Max gen: 19, Max flush: 31 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.541066247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.549852548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550177001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550203462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550214752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550226872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Crucible 76f8ad62-fb65-4790-942e-09c327211e5d has session id: 98b26d5b-0889-474e-9e76-e619ae51511e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550614466Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Upstairs opts: Upstairs UUID: 76f8ad62-fb65-4790-942e-09c327211e5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550658017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Crucible stats registered with UUID: 76f8ad62-fb65-4790-942e-09c327211e5d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550669237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:03.550678667Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.550872499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.55090293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.55091575Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551015701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551042681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"76f8ad62-fb65-4790-942e-09c327211e5d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551052061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551061671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551071931Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551084212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551094402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551104912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551115412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60970)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551164883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48695)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551191093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48996)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551202853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.55184331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551866601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.551917831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552201944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552213785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[0]R flush_numbers[0..12]: [21, 12, 31, 26, 30, 30, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552251485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[0]R generation[0..12]: [13, 6, 19, 14, 18, 18, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552263165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552274975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[1]R flush_numbers[0..12]: [21, 12, 31, 26, 30, 30, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552288465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[1]R generation[0..12]: [13, 6, 19, 14, 18, 18, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552299376Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552310886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[2]R flush_numbers[0..12]: [21, 12, 31, 26, 30, 30, 24, 1, 26, 26, 15, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552322056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[2]R generation[0..12]: [13, 6, 19, 14, 18, 18, 14, 1, 14, 14, 9, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552333056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552344616Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552354456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552364646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552374496Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552384487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"":"downstairs","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552394827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552404247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"0","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552414627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"1","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552424907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"client":"2","":"downstairs","session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552435167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552446157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"76f8ad62-fb65-4790-942e-09c327211e5d is now active with session: 98b26d5b-0889-474e-9e76-e619ae51511e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552456637Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552465827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"session_id":"98b26d5b-0889-474e-9e76-e619ae51511e"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552475368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Activated sub_volume 76f8ad62-fb65-4790-942e-09c327211e5d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552483838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552492378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.552861692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} 01/30 Write block 567 len 24576 data: 3 4 3 3 3 4 02/30 Read block 566 len 12288 03/30 Read block 260 len 28672 04/30 Read block 165 len 40960 05/30 Read block 523 len 12288 06/30 Write block 31 len 12288 data: 2 2 2 07/30 Write block 13 len 20480 data: 4 4 3 3 3 08/30 Read block 412 len 24576 09/30 Read block 429 len 12288 10/30 Write block 434 len 28672 data: 4 5 5 6 6 6 4 11/30 Read block 327 len 24576 12/30 Write block 204 len 8192 data: 2 2 13/30 Write block 572 len 40960 data: 5 6 6 7 7 5 6 7 6 5 14/30 Read block 247 len 40960 15/30 Read block 145 len 16384 16/30 Write block 351 len 16384 data: 3 2 2 2 17/30 Read block 534 len 4096 18/30 Read block 140 len 24576 19/30 Read block 496 len 32768 20/30 Write block 284 len 28672 data: 2 2 2 2 2 2 2 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 204 len 4096 data: 3 25/30 Read block 149 len 16384 26/30 Write block 332 len 20480 data: 5 5 4 3 3 27/30 Write block 414 len 32768 data: 2 3 3 3 2 2 2 2 28/30 Read block 334 len 20480 29/30 Read block 433 len 8192 30/30 Read block 176 len 12288 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:03.561957836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:03.562229419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 13 20 32 21 32 F F F <--- 1 020-039 20 6 20 32 12 32 F F F <--- 2 040-059 19 19 19 31 31 31 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 18 18 18 30 30 30 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 20 9 20 32 15 32 T F T <--- 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 19 19 19 31 31 31 F F F 14 280-299 20 17 20 32 29 32 F F F <--- 15 300-319 17 17 17 29 29 29 F F F 16 320-339 10 10 10 17 17 17 T F T <--- 17 340-359 20 16 20 32 28 32 F F F <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 17 17 17 29 29 29 T F T <--- 21 420-439 20 18 20 32 30 32 T F T <--- 22 440-459 20 16 20 32 28 32 F F F <--- 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 20 18 20 32 30 32 F T F <--- 29 580-599 20 16 20 32 28 32 F F F <--- Max gen: 20, Max flush: 32 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 20 20 20 32 32 32 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 14 14 14 26 26 26 F F F 4 080-099 18 18 18 30 30 30 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F F F 10 200-219 21 21 21 33 33 33 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 19 19 19 31 31 31 F F F 14 280-299 20 20 20 32 32 32 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 21 21 21 33 33 33 F F F 17 340-359 20 20 20 32 32 32 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 21 21 21 33 33 33 F F F 21 420-439 21 21 21 33 33 33 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 11 11 19 19 19 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 20 20 20 32 32 32 F F F 29 580-599 20 20 20 32 32 32 F F F Max gen: 21, Max flush: 33 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.80656461Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:07.814046296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.814125127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.821433779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.821694932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.821728003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.821736863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.821748683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Crucible 81c1503d-d960-459b-aecf-7dcbff92245a has session id: 945fd8c4-47c7-487f-8d83-a7bb5e70ac78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822149338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Upstairs opts: Upstairs UUID: 81c1503d-d960-459b-aecf-7dcbff92245a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822182368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Crucible stats registered with UUID: 81c1503d-d960-459b-aecf-7dcbff92245a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822195388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:07.822205638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.82238589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.82240227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822412891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822550282Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822568752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"81c1503d-d960-459b-aecf-7dcbff92245a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822592483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822602253Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822612473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822622543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822632593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822643163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822653643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"ds_connection connected from Ok(127.0.0.1:43740)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822792645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"ds_connection connected from Ok(127.0.0.1:37945)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822894346Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"ds_connection connected from Ok(127.0.0.1:37931)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.822907936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823439482Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823521483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823546894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823748296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823766836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[0]R flush_numbers[0..12]: [32, 32, 31, 26, 30, 30, 24, 1, 26, 26, 33, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823843817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[0]R generation[0..12]: [20, 20, 19, 14, 18, 18, 14, 1, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823855307Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823866887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[1]R flush_numbers[0..12]: [32, 32, 31, 26, 30, 30, 24, 1, 26, 26, 33, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823877717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[1]R generation[0..12]: [20, 20, 19, 14, 18, 18, 14, 1, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823888507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823899758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[2]R flush_numbers[0..12]: [32, 32, 31, 26, 30, 30, 24, 1, 26, 26, 33, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823910628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[2]R generation[0..12]: [20, 20, 19, 14, 18, 18, 14, 1, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823923848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823935068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823944738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823954888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823964688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823974448Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"":"downstairs","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823984609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.823993809Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"0","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824003979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"1","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824014109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"client":"2","":"downstairs","session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824024419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824033949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"81c1503d-d960-459b-aecf-7dcbff92245a is now active with session: 945fd8c4-47c7-487f-8d83-a7bb5e70ac78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824044429Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824053589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"session_id":"945fd8c4-47c7-487f-8d83-a7bb5e70ac78"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824063239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Activated sub_volume 81c1503d-d960-459b-aecf-7dcbff92245a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.82407174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.82408042Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.824503054Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} 01/30 Write block 267 len 28672 data: 4 3 3 3 3 2 2 02/30 Read block 464 len 4096 03/30 Read block 258 len 12288 04/30 Write block 296 len 40960 data: 2 2 2 3 3 3 3 4 4 3 05/30 Read block 114 len 24576 06/30 Read block 307 len 40960 07/30 Write block 483 len 8192 data: 2 2 08/30 Write block 211 len 4096 data: 3 09/30 Write block 573 len 36864 data: 7 7 8 8 6 7 8 7 6 10/30 Read block 384 len 16384 11/30 Read block 270 len 16384 12/30 Read block 13 len 12288 13/30 Read block 273 len 36864 14/30 Write block 483 len 12288 data: 3 3 2 15/30 Read block 505 len 4096 16/30 Write block 26 len 16384 data: 2 2 2 2 17/30 Read block 145 len 32768 18/30 Write block 58 len 40960 data: 2 3 3 3 3 3 3 3 3 3 19/30 Write block 46 len 16384 data: 2 2 3 3 20/30 Write block 311 len 28672 data: 4 4 4 3 3 5 4 21/30 Flush 22/30 Write block 41 len 32768 data: 2 2 2 2 3 3 3 4 23/30 Write block 489 len 16384 data: 3 2 2 2 24/30 Read block 16 len 36864 25/30 Read block 411 len 4096 26/30 Read block 124 len 28672 27/30 Write block 100 len 24576 data: 4 4 4 5 5 7 28/30 Write block 183 len 20480 data: 3 3 4 4 4 29/30 Write block 278 len 4096 data: 4 30/30 Write block 326 len 16384 data: 2 2 2 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 4 4 38 Done 0 26 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:07.833085173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:07.833366806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 20 22 22 32 34 34 F F F <--- 2 040-059 19 22 22 31 34 34 F T T <--- 3 060-079 14 22 22 26 34 34 F F F <--- 4 080-099 18 18 18 30 30 30 F F F 5 100-119 18 18 18 30 30 30 F T T <--- 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 14 14 14 26 26 26 F T T <--- 10 200-219 21 22 22 33 34 34 F F F <--- 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 19 22 22 31 34 34 F T T <--- 14 280-299 20 22 22 32 34 34 F F F <--- 15 300-319 17 22 22 29 34 34 F F F <--- 16 320-339 21 21 21 33 33 33 F T T <--- 17 340-359 20 20 20 32 32 32 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 21 21 21 33 33 33 F F F 21 420-439 21 21 21 33 33 33 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 11 22 22 19 34 34 F T T <--- 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 20 22 22 32 34 34 F F F <--- 29 580-599 20 22 22 32 34 34 F F F <--- Max gen: 22, Max flush: 34 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 22 22 22 34 34 34 F F F 4 080-099 18 18 18 30 30 30 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 23 23 23 35 35 35 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 23 23 23 35 35 35 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 22 22 22 34 34 34 F F F 16 320-339 23 23 23 35 35 35 F F F 17 340-359 20 20 20 32 32 32 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 F F F 20 400-419 21 21 21 33 33 33 F F F 21 420-439 21 21 21 33 33 33 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 23, Max flush: 35 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.087084834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.095857505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096159908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096183868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096192969Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096204959Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Crucible de7e595c-d0ee-42c9-bc36-328a1bc1dec9 has session id: 37b904e0-960b-4be4-a4bc-74d755de7613","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096591803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Upstairs opts: Upstairs UUID: de7e595c-d0ee-42c9-bc36-328a1bc1dec9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096634744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Crucible stats registered with UUID: de7e595c-d0ee-42c9-bc36-328a1bc1dec9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096645874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:12.096655494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096851186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096881036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.096892397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097054638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097070669Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"de7e595c-d0ee-42c9-bc36-328a1bc1dec9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097086819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097096369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097106649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097116829Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097127039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097137379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097147749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"ds_connection connected from Ok(127.0.0.1:57245)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.0972372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"ds_connection connected from Ok(127.0.0.1:41577)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097286121Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"ds_connection connected from Ok(127.0.0.1:47823)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097331662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097953119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.097971629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.09803111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098322713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098335683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[0]R flush_numbers[0..12]: [32, 34, 35, 34, 30, 35, 24, 1, 26, 35, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098362403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[0]R generation[0..12]: [20, 22, 23, 22, 18, 23, 14, 1, 14, 23, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098374093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098385694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[1]R flush_numbers[0..12]: [32, 34, 35, 34, 30, 35, 24, 1, 26, 35, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098396634Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[1]R generation[0..12]: [20, 22, 23, 22, 18, 23, 14, 1, 14, 23, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098407674Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098419324Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[2]R flush_numbers[0..12]: [32, 34, 35, 34, 30, 35, 24, 1, 26, 35, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098430414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[2]R generation[0..12]: [20, 22, 23, 22, 18, 23, 14, 1, 14, 23, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098441294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098452864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098462914Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098473375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098483255Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098493425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"":"downstairs","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098505085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098514505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"0","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098525135Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"1","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098535865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"client":"2","":"downstairs","session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098546625Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098556426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"de7e595c-d0ee-42c9-bc36-328a1bc1dec9 is now active with session: 37b904e0-960b-4be4-a4bc-74d755de7613","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098566946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098576316Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"session_id":"37b904e0-960b-4be4-a4bc-74d755de7613"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098586066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Activated sub_volume de7e595c-d0ee-42c9-bc36-328a1bc1dec9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098594636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.098603296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.099039181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} 01/30 Write block 389 len 16384 data: 3 3 4 4 02/30 Read block 472 len 16384 03/30 Write block 82 len 4096 data: 4 04/30 Write block 180 len 8192 data: 5 4 05/30 Read block 428 len 12288 06/30 Read block 399 len 32768 07/30 Write block 436 len 28672 data: 6 7 7 7 5 4 4 08/30 Write block 333 len 8192 data: 6 5 09/30 Write block 407 len 32768 data: 3 2 2 3 3 3 3 3 10/30 Read block 16 len 4096 11/30 Write block 578 len 40960 data: 8 9 8 7 6 5 6 5 5 5 12/30 Write block 561 len 32768 data: 3 4 4 4 4 4 4 5 13/30 Read block 441 len 16384 14/30 Read block 542 len 32768 15/30 Write block 450 len 4096 data: 3 16/30 Write block 253 len 16384 data: 3 3 4 6 17/30 Read block 559 len 20480 18/30 Read block 313 len 36864 19/30 Read block 302 len 40960 20/30 Write block 81 len 32768 data: 4 5 2 3 3 3 3 3 21/30 Write block 275 len 16384 data: 4 4 4 5 22/30 Read block 339 len 36864 23/30 Read block 405 len 4096 24/30 Write block 190 len 8192 data: 3 4 25/30 Read block 257 len 12288 26/30 Read block 377 len 20480 27/30 Read block 197 len 28672 28/30 Read block 449 len 40960 29/30 Read block 501 len 28672 30/30 Write block 544 len 16384 data: 5 6 8 8 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:12.107192374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:12.107467358Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 22 22 22 34 34 34 F F F 4 080-099 18 18 18 30 30 30 T F T <--- 5 100-119 23 23 23 35 35 35 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 23 23 23 35 35 35 T F T <--- 10 200-219 22 22 22 34 34 34 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 19 19 19 31 31 31 T F T <--- 13 260-279 23 23 23 35 35 35 T F T <--- 14 280-299 22 22 22 34 34 34 F F F 15 300-319 22 22 22 34 34 34 F F F 16 320-339 23 23 23 35 35 35 T F T <--- 17 340-359 20 20 20 32 32 32 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 17 17 17 29 29 29 T F T <--- 20 400-419 21 21 21 33 33 33 T F T <--- 21 420-439 21 21 21 33 33 33 T F T <--- 22 440-459 20 20 20 32 32 32 T F T <--- 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 19 19 19 31 31 31 T F T <--- 28 560-579 22 22 22 34 34 34 T F T <--- 29 580-599 22 22 22 34 34 34 T F T <--- Max gen: 23, Max flush: 35 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 23 23 23 35 35 35 F F F 3 060-079 22 22 22 34 34 34 F F F 4 080-099 24 24 24 36 36 36 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 24 24 24 36 36 36 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 14 14 14 22 22 22 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 36 36 36 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 22 22 22 34 34 34 F F F 16 320-339 24 24 24 36 36 36 F F F 17 340-359 20 20 20 32 32 32 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 24 24 24 36 36 36 F F F 20 400-419 24 24 24 36 36 36 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 24 24 24 36 36 36 F F F 28 560-579 24 24 24 36 36 36 F F F 29 580-599 24 24 24 36 36 36 F F F Max gen: 24, Max flush: 36 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.354213356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.362766494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363051117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363075217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363084258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363098158Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Crucible 7d40ab04-9754-43af-88fb-1e38b392aa1c has session id: 2b409a2e-a836-4ce6-b725-f0aeacbfd6c9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363537783Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Upstairs opts: Upstairs UUID: 7d40ab04-9754-43af-88fb-1e38b392aa1c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363577353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Crucible stats registered with UUID: 7d40ab04-9754-43af-88fb-1e38b392aa1c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363588083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:16.363597463Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363771445Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363786686Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363797086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363942757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363961448Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"7d40ab04-9754-43af-88fb-1e38b392aa1c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363985308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.363994818Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364004958Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364015208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364025068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364037019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364047229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33748)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364111389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56210)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364122729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37040)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364211321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364838068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364859638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.364922809Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365170611Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365180452Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[0]R flush_numbers[0..12]: [32, 34, 35, 34, 36, 35, 24, 1, 26, 36, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365323833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[0]R generation[0..12]: [20, 22, 23, 22, 24, 23, 14, 1, 14, 24, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365335713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365347204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[1]R flush_numbers[0..12]: [32, 34, 35, 34, 36, 35, 24, 1, 26, 36, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365358094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[1]R generation[0..12]: [20, 22, 23, 22, 24, 23, 14, 1, 14, 24, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365370904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365382294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[2]R flush_numbers[0..12]: [32, 34, 35, 34, 36, 35, 24, 1, 26, 36, 34, 22]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365393134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[2]R generation[0..12]: [20, 22, 23, 22, 24, 23, 14, 1, 14, 24, 22, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365403744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365414994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Max found gen is 25","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365424764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Generation requested: 26 >= found:25","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365434695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365444515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365454215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"":"downstairs","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365464405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365473695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"0","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365483865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"1","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365493905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"client":"2","":"downstairs","session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365504185Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365513635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"7d40ab04-9754-43af-88fb-1e38b392aa1c is now active with session: 2b409a2e-a836-4ce6-b725-f0aeacbfd6c9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365523806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365533946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"session_id":"2b409a2e-a836-4ce6-b725-f0aeacbfd6c9"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365543356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Activated sub_volume 7d40ab04-9754-43af-88fb-1e38b392aa1c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365551606Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.365560146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.36595272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} 01/30 Write block 339 len 28672 data: 3 3 4 4 3 3 3 02/30 Write block 257 len 12288 data: 5 6 6 03/30 Write block 258 len 32768 data: 7 7 5 6 6 5 5 5 04/30 Read block 285 len 24576 05/30 Read block 184 len 12288 06/30 Flush 07/30 Write block 72 len 16384 data: 3 3 3 4 08/30 Read block 384 len 28672 09/30 Read block 386 len 4096 10/30 Write block 303 len 28672 data: 5 5 4 3 3 3 3 11/30 Flush 12/30 Flush 13/30 Write block 223 len 4096 data: 2 14/30 Write block 365 len 20480 data: 3 2 2 2 2 15/30 Read block 28 len 24576 16/30 Write block 59 len 32768 data: 4 4 4 4 4 4 4 4 17/30 Write block 235 len 32768 data: 3 3 3 4 5 3 3 4 18/30 Write block 258 len 32768 data: 8 8 6 7 7 6 6 6 19/30 Flush 20/30 Write block 585 len 12288 data: 6 6 6 21/30 Read block 170 len 12288 22/30 Write block 556 len 36864 data: 2 2 4 4 3 4 5 5 5 23/30 Read block 220 len 28672 24/30 Write block 576 len 28672 data: 9 7 9 10 9 8 7 25/30 Read block 239 len 28672 26/30 Read block 399 len 12288 27/30 Read block 409 len 40960 28/30 Write block 43 len 4096 data: 3 29/30 Read block 361 len 4096 30/30 Read block 375 len 24576 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 21 22 Done 24 23 3 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:16.374740661Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:16.375015224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 26 26 23 40 40 35 T T F <--- 3 060-079 26 26 22 40 40 34 F F T <--- 4 080-099 24 24 24 36 36 36 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 24 24 24 36 36 36 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 26 26 14 40 40 22 F F F <--- 12 240-259 26 26 26 40 40 37 F F F <--- 13 260-279 26 26 26 40 40 37 F F F <--- 14 280-299 22 22 22 34 34 34 F F F 15 300-319 26 26 22 38 38 34 F F F <--- 16 320-339 26 26 26 37 37 37 F F F 17 340-359 26 26 26 37 37 37 F F F 18 360-379 26 26 2 40 40 2 F F F <--- 19 380-399 24 24 24 36 36 36 F F F 20 400-419 24 24 24 36 36 36 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 24 24 24 36 36 36 T T F <--- 28 560-579 24 24 24 36 36 36 T T F <--- 29 580-599 24 24 24 36 36 36 T T F <--- Max gen: 26, Max flush: 40 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 27 27 27 41 41 41 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 24 24 24 36 36 36 F F F 5 100-119 23 23 23 35 35 35 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 24 24 24 36 36 36 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 26 26 26 40 40 40 F F F 12 240-259 26 26 26 40 40 40 F F F 13 260-279 26 26 26 40 40 40 F F F 14 280-299 22 22 22 34 34 34 F F F 15 300-319 26 26 26 38 38 38 F F F 16 320-339 26 26 26 37 37 37 F F F 17 340-359 26 26 26 37 37 37 F F F 18 360-379 26 26 26 40 40 40 F F F 19 380-399 24 24 24 36 36 36 F F F 20 400-419 24 24 24 36 36 36 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 27 27 27 41 41 41 F F F 28 560-579 27 27 27 41 41 41 F F F 29 580-599 27 27 27 41 41 41 F F F Max gen: 27, Max flush: 41 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.612992402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.621470389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.621770203Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.621794393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.621803633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.621815663Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Crucible d7c7ecb3-3a30-4982-be13-bd3fb9754a80 has session id: c7168227-f2fa-411d-92be-3cb90f45bdf1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622205628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Upstairs opts: Upstairs UUID: d7c7ecb3-3a30-4982-be13-bd3fb9754a80, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622236298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Crucible stats registered with UUID: d7c7ecb3-3a30-4982-be13-bd3fb9754a80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622248218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:20.622258268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622449131Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622481411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622493061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622658193Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622683793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"d7c7ecb3-3a30-4982-be13-bd3fb9754a80 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622698013Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622707423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622717804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622728164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622738244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622748964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622759544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62381)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622832125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43413)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622858805Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43197)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.622968276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.623646984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.623659804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.623673735Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.623992328Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624002398Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[0]R flush_numbers[0..12]: [32, 34, 41, 40, 36, 35, 24, 1, 26, 36, 34, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624045079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[0]R generation[0..12]: [20, 22, 27, 26, 24, 23, 14, 1, 14, 24, 22, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624056689Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624068279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[1]R flush_numbers[0..12]: [32, 34, 41, 40, 36, 35, 24, 1, 26, 36, 34, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624079239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[1]R generation[0..12]: [20, 22, 27, 26, 24, 23, 14, 1, 14, 24, 22, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624090179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624101779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[2]R flush_numbers[0..12]: [32, 34, 41, 40, 36, 35, 24, 1, 26, 36, 34, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62411289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[2]R generation[0..12]: [20, 22, 27, 26, 24, 23, 14, 1, 14, 24, 22, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62412484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62413639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.6241465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62415778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.6241691Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62417904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"":"downstairs","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.62418929Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624198511Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"0","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624208641Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"1","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624218761Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"client":"2","":"downstairs","session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624229001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624238611Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"d7c7ecb3-3a30-4982-be13-bd3fb9754a80 is now active with session: c7168227-f2fa-411d-92be-3cb90f45bdf1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624248981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624258241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"session_id":"c7168227-f2fa-411d-92be-3cb90f45bdf1"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624267871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Activated sub_volume d7c7ecb3-3a30-4982-be13-bd3fb9754a80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624276471Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624285232Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.624669876Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} 01/30 Write block 109 len 12288 data: 6 5 5 02/30 Read block 513 len 24576 03/30 Read block 372 len 4096 04/30 Write block 276 len 28672 data: 5 5 6 3 3 2 2 05/30 Read block 230 len 12288 06/30 Flush 07/30 Write block 354 len 32768 data: 3 2 2 2 2 2 2 2 08/30 Write block 534 len 4096 data: 3 09/30 Write block 192 len 32768 data: 5 5 5 4 3 3 3 2 10/30 Write block 559 len 40960 data: 5 4 5 6 6 6 5 5 5 6 11/30 Read block 293 len 16384 12/30 Read block 127 len 36864 13/30 Write block 94 len 16384 data: 4 5 6 6 14/30 Read block 226 len 4096 15/30 Write block 398 len 28672 data: 4 3 4 3 3 3 5 16/30 Read block 368 len 24576 17/30 Write block 201 len 20480 data: 2 2 2 4 3 18/30 Write block 385 len 12288 data: 4 3 4 19/30 Write block 398 len 36864 data: 5 4 5 4 4 4 6 4 3 20/30 Write block 216 len 8192 data: 2 2 21/30 Write block 317 len 24576 data: 5 4 4 3 3 3 22/30 Read block 318 len 36864 23/30 Read block 404 len 36864 24/30 Write block 211 len 4096 data: 4 25/30 Read block 22 len 4096 26/30 Flush 27/30 Write block 58 len 8192 data: 3 5 28/30 Read block 165 len 40960 29/30 Write block 163 len 24576 data: 3 3 3 3 3 3 30/30 Write block 25 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 28 3 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:20.633040522Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:20.633324895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 22 22 22 34 34 34 T F T <--- 2 040-059 27 27 27 41 41 41 T F T <--- 3 060-079 26 26 26 40 40 40 F F F 4 080-099 28 24 28 43 36 43 F F F <--- 5 100-119 28 23 28 42 35 42 F T F <--- 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 14 14 14 26 26 26 T F T <--- 9 180-199 28 24 28 43 36 43 F F F <--- 10 200-219 28 22 28 43 34 43 F F F <--- 11 220-239 26 26 26 40 40 40 F F F 12 240-259 26 26 26 40 40 40 F F F 13 260-279 28 26 28 42 40 42 F F F <--- 14 280-299 28 22 28 42 34 42 F F F <--- 15 300-319 28 26 28 43 38 43 F F F <--- 16 320-339 28 26 28 43 37 43 F F F <--- 17 340-359 28 26 28 43 37 43 F F F <--- 18 360-379 28 26 28 43 40 43 F F F <--- 19 380-399 28 24 28 43 36 43 F F F <--- 20 400-419 28 24 28 43 36 43 F F F <--- 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 19 28 43 31 43 F F F <--- 27 540-559 28 27 28 43 41 43 F F F <--- 28 560-579 28 27 28 43 41 43 F F F <--- 29 580-599 27 27 27 41 41 41 F F F Max gen: 28, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 29 29 29 44 44 44 F F F 3 060-079 26 26 26 40 40 40 F F F 4 080-099 28 28 28 43 43 43 F F F 5 100-119 28 28 28 42 42 42 F F F 6 120-139 14 14 14 24 24 24 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 29 29 29 44 44 44 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 F F F 11 220-239 26 26 26 40 40 40 F F F 12 240-259 26 26 26 40 40 40 F F F 13 260-279 28 28 28 42 42 42 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 28 28 28 43 43 43 F F F 20 400-419 28 28 28 43 43 43 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 23 23 23 35 35 35 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 28 28 28 43 43 43 F F F 28 560-579 28 28 28 43 43 43 F F F 29 580-599 27 27 27 41 41 41 F F F Max gen: 29, Max flush: 44 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.890944957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.899475835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.899762308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.899786038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.899796878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.899808899Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Crucible e5e41aec-f84e-41da-bf0e-6416c4e35594 has session id: b8561443-5ad1-410e-b9db-cfb06d5c18df","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900199213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Upstairs opts: Upstairs UUID: e5e41aec-f84e-41da-bf0e-6416c4e35594, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900228493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Crucible stats registered with UUID: e5e41aec-f84e-41da-bf0e-6416c4e35594","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900239983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:24.900249774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900431686Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900447736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900458546Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900600638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900624418Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"e5e41aec-f84e-41da-bf0e-6416c4e35594 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900668188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900678388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900689209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900701639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900711609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900722029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900732239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"ds_connection connected from Ok(127.0.0.1:62817)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.90080197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43243)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.90081427Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"ds_connection connected from Ok(127.0.0.1:58286)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.900903421Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901490018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901508058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901566769Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901818122Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901844192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[0]R flush_numbers[0..12]: [32, 44, 44, 40, 43, 42, 24, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901897212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[0]R generation[0..12]: [20, 29, 29, 26, 28, 28, 14, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901909313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901921123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[1]R flush_numbers[0..12]: [32, 44, 44, 40, 43, 42, 24, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901934633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[1]R generation[0..12]: [20, 29, 29, 26, 28, 28, 14, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901945393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901956783Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[2]R flush_numbers[0..12]: [32, 44, 44, 40, 43, 42, 24, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901967493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[2]R generation[0..12]: [20, 29, 29, 26, 28, 28, 14, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901978183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.901989623Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902013494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902023594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902033384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902043104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"":"downstairs","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902053374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902062694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"0","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902072744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"1","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902082715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"client":"2","":"downstairs","session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902092825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902108245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"e5e41aec-f84e-41da-bf0e-6416c4e35594 is now active with session: b8561443-5ad1-410e-b9db-cfb06d5c18df","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902118545Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902127465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"session_id":"b8561443-5ad1-410e-b9db-cfb06d5c18df"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902136935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Activated sub_volume e5e41aec-f84e-41da-bf0e-6416c4e35594","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902145215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.902153745Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.90255347Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} 01/30 Write block 540 len 28672 data: 4 6 5 5 6 7 9 02/30 Read block 213 len 20480 03/30 Write block 242 len 4096 data: 5 04/30 Write block 53 len 36864 data: 3 3 4 4 3 4 6 5 5 05/30 Flush 06/30 Write block 248 len 24576 data: 3 2 3 3 3 4 07/30 Write block 584 len 12288 data: 7 7 7 08/30 Read block 563 len 16384 09/30 Write block 374 len 40960 data: 2 2 2 2 2 2 2 2 4 4 10/30 Write block 499 len 4096 data: 3 11/30 Write block 126 len 20480 data: 2 4 4 4 4 12/30 Read block 180 len 20480 13/30 Write block 490 len 36864 data: 3 3 3 2 2 2 3 3 3 14/30 Flush 15/30 Flush 16/30 Write block 282 len 36864 data: 3 2 3 3 3 3 3 3 3 17/30 Read block 88 len 36864 18/30 Read block 512 len 28672 19/30 Read block 353 len 16384 20/30 Write block 110 len 40960 data: 6 6 4 5 3 2 2 2 2 2 21/30 Read block 16 len 28672 22/30 Flush 23/30 Write block 558 len 12288 data: 5 6 5 24/30 Read block 544 len 36864 25/30 Read block 208 len 20480 26/30 Read block 24 len 20480 27/30 Read block 580 len 36864 28/30 Write block 121 len 28672 data: 2 2 2 2 2 3 5 29/30 Write block 365 len 12288 data: 4 3 3 30/30 Read block 81 len 16384 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 29 31 Done 28 30 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:24.911566753Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:24.911861836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 29 45 45 44 F F F <--- 3 060-079 30 30 26 45 45 40 F F F <--- 4 080-099 28 28 28 43 43 43 F F F 5 100-119 30 30 28 48 48 42 F F F <--- 6 120-139 30 30 14 46 46 24 T T F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 29 29 29 44 44 44 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 F F F 11 220-239 26 26 26 40 40 40 F F F 12 240-259 30 30 26 46 46 40 F F F <--- 13 260-279 28 28 28 42 42 42 F F F 14 280-299 30 30 28 48 48 42 F F F <--- 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 30 30 28 46 46 43 T T F <--- 19 380-399 30 30 28 46 46 43 F F F <--- 20 400-419 28 28 28 43 43 43 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 30 30 23 46 46 35 F F F <--- 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 30 30 28 45 45 43 T T T <--- 28 560-579 28 28 28 43 43 43 T T F <--- 29 580-599 30 30 27 46 46 41 F F F <--- Max gen: 30, Max flush: 48 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 30 45 45 45 F F F 3 060-079 30 30 30 45 45 45 F F F 4 080-099 28 28 28 43 43 43 F F F 5 100-119 30 30 30 48 48 48 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 29 29 29 44 44 44 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 F F F 11 220-239 26 26 26 40 40 40 F F F 12 240-259 30 30 30 46 46 46 F F F 13 260-279 28 28 28 42 42 42 F F F 14 280-299 30 30 30 48 48 48 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 30 30 30 46 46 46 F F F 20 400-419 28 28 28 43 43 43 F F F 21 420-439 24 24 24 36 36 36 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 30 30 30 46 46 46 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 31 31 31 49 49 49 F F F 28 560-579 31 31 31 49 49 49 F F F 29 580-599 30 30 30 46 46 46 F F F Max gen: 31, Max flush: 49 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.157741645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.166344044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.166628847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.166654807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.166663987Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.166676027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Crucible 00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9 has session id: a2f29b14-055e-4f5d-b190-083cef64737a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167055842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Upstairs opts: Upstairs UUID: 00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167094902Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Crucible stats registered with UUID: 00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167107552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:29.167117022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167299944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167329755Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167341375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167479107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167505857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167518937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167528317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167538147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167548047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167558027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167568438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167578828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49531)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167612928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48025)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167624408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51456)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.167734249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168341146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168368347Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168498728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.16869112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168701881Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[0]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 48, 49, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168847852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[0]R generation[0..12]: [20, 29, 30, 30, 28, 30, 31, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168859462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168871012Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[1]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 48, 49, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168881973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[1]R generation[0..12]: [20, 29, 30, 30, 28, 30, 31, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168892543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168903823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[2]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 48, 49, 1, 44, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168914613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[2]R generation[0..12]: [20, 29, 30, 30, 28, 30, 31, 1, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168925293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168938473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168948383Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168958583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168968084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168977864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"":"downstairs","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168988104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.168997494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"0","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169007974Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"1","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169018194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"client":"2","":"downstairs","session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169028534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169038194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9 is now active with session: a2f29b14-055e-4f5d-b190-083cef64737a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169048554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169057875Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"session_id":"a2f29b14-055e-4f5d-b190-083cef64737a"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169067365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Activated sub_volume 00ab4ff6-86f3-4583-9cf1-8bbcfac33cd9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169075855Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.169084305Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.16951359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} 01/30 Write block 273 len 24576 data: 3 3 5 6 6 7 02/30 Read block 250 len 20480 03/30 Write block 437 len 12288 data: 8 8 8 04/30 Read block 13 len 4096 05/30 Read block 512 len 16384 06/30 Write block 444 len 16384 data: 2 2 2 2 07/30 Read block 183 len 4096 08/30 Write block 224 len 40960 data: 2 2 2 2 2 2 2 2 3 3 09/30 Read block 49 len 40960 10/30 Write block 480 len 32768 data: 2 2 2 4 4 3 2 2 11/30 Write block 426 len 24576 data: 2 3 3 3 4 4 12/30 Write block 138 len 36864 data: 3 2 2 2 2 2 2 2 2 13/30 Flush 14/30 Write block 227 len 40960 data: 3 3 3 3 3 4 4 3 4 4 15/30 Write block 455 len 16384 data: 3 3 3 3 16/30 Write block 217 len 4096 data: 3 17/30 Read block 121 len 32768 18/30 Write block 557 len 4096 data: 3 19/30 Write block 397 len 40960 data: 4 6 5 6 5 5 5 7 5 4 20/30 Flush 21/30 Write block 158 len 28672 data: 2 2 2 3 3 4 4 22/30 Read block 104 len 4096 23/30 Read block 78 len 20480 24/30 Read block 235 len 16384 25/30 Read block 98 len 28672 26/30 Read block 65 len 28672 27/30 Read block 125 len 32768 28/30 Write block 105 len 20480 data: 8 8 7 6 7 29/30 Read block 451 len 20480 30/30 Read block 578 len 4096 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 30 30 6 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:29.178456712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:29.178721315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 30 45 45 45 F F F 3 060-079 30 30 30 45 45 45 F F F 4 080-099 28 28 28 43 43 43 F F F 5 100-119 30 30 30 48 48 48 T T F <--- 6 120-139 32 32 31 50 50 49 F F F <--- 7 140-159 32 32 1 50 50 1 T T F <--- 8 160-179 29 29 29 44 44 44 T T F <--- 9 180-199 28 28 28 43 43 43 F F F 10 200-219 32 32 28 51 51 43 F F F <--- 11 220-239 32 32 26 51 51 40 F F F <--- 12 240-259 30 30 30 46 46 46 F F F 13 260-279 32 32 28 50 50 42 F F T <--- 14 280-299 30 30 30 48 48 48 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 32 32 30 51 51 46 F F F <--- 20 400-419 32 32 28 51 51 43 F F F <--- 21 420-439 32 32 24 50 50 36 F F T <--- 22 440-459 32 32 24 51 51 36 F F T <--- 23 460-479 15 15 15 27 27 27 F F F 24 480-499 32 32 30 50 50 46 F F F <--- 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 32 32 31 51 51 49 F F F <--- 28 560-579 31 31 31 49 49 49 F F F 29 580-599 30 30 30 46 46 46 F F F Max gen: 32, Max flush: 51 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 30 45 45 45 F F F 3 060-079 30 30 30 45 45 45 F F F 4 080-099 28 28 28 43 43 43 F F F 5 100-119 33 33 33 52 52 52 F F F 6 120-139 32 32 32 50 50 50 F F F 7 140-159 33 33 33 52 52 52 F F F 8 160-179 33 33 33 52 52 52 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 30 30 30 46 46 46 F F F 13 260-279 32 32 32 50 50 50 F F F 14 280-299 30 30 30 48 48 48 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 32 32 32 51 51 51 F F F 20 400-419 32 32 32 51 51 51 F F F 21 420-439 32 32 32 50 50 50 F F F 22 440-459 32 32 32 51 51 51 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 32 32 32 50 50 50 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 32 32 32 51 51 51 F F F 28 560-579 31 31 31 49 49 49 F F F 29 580-599 30 30 30 46 46 46 F F F Max gen: 33, Max flush: 52 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.424807366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.433503736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.43384052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.4338658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.43387576Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.43388772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Crucible c9fc660b-80d4-46c0-891b-4ed19b9ad406 has session id: d7c899dc-0f70-45fd-9287-37a4a1d6f1c0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434218724Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Upstairs opts: Upstairs UUID: c9fc660b-80d4-46c0-891b-4ed19b9ad406, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434248524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Crucible stats registered with UUID: c9fc660b-80d4-46c0-891b-4ed19b9ad406","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434260284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:33.434270584Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434456807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434487137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434498427Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434613138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434634619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"c9fc660b-80d4-46c0-891b-4ed19b9ad406 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434661309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434671099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434681229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434691379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434701409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.43471176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.43472245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:37092)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434798831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53593)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434811141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:60993)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.434821891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435465098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435536129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435558459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435742901Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435807742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[0]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 52, 50, 52, 52, 43, 51, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435885643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[0]R generation[0..12]: [20, 29, 30, 30, 28, 33, 32, 33, 33, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435897333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435908673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[1]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 52, 50, 52, 52, 43, 51, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435919333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[1]R generation[0..12]: [20, 29, 30, 30, 28, 33, 32, 33, 33, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435929863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435941074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[2]R flush_numbers[0..12]: [32, 44, 45, 45, 43, 52, 50, 52, 52, 43, 51, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435951894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[2]R generation[0..12]: [20, 29, 30, 30, 28, 33, 32, 33, 33, 28, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435962464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435973744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435983704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.435993684Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436003334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436013184Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"":"downstairs","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436023205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436033915Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"0","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436043925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"1","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436053875Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"client":"2","":"downstairs","session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436064045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436073585Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"c9fc660b-80d4-46c0-891b-4ed19b9ad406 is now active with session: d7c899dc-0f70-45fd-9287-37a4a1d6f1c0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436083905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436093005Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"session_id":"d7c899dc-0f70-45fd-9287-37a4a1d6f1c0"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436102445Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Activated sub_volume c9fc660b-80d4-46c0-891b-4ed19b9ad406","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436111096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436127096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.436594331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} 01/30 Flush 02/30 Write block 430 len 40960 data: 5 5 5 5 5 6 7 9 9 9 03/30 Read block 206 len 32768 04/30 Write block 574 len 36864 data: 8 9 10 8 10 11 10 9 8 05/30 Write block 208 len 20480 data: 3 3 3 5 3 06/30 Read block 406 len 20480 07/30 Read block 458 len 32768 08/30 Write block 500 len 12288 data: 4 4 3 09/30 Write block 258 len 24576 data: 9 9 7 8 8 7 10/30 Write block 78 len 24576 data: 4 3 4 5 6 3 11/30 Read block 548 len 20480 12/30 Write block 190 len 32768 data: 4 5 6 6 6 5 4 4 13/30 Write block 165 len 36864 data: 4 4 4 4 3 3 2 2 3 14/30 Write block 263 len 4096 data: 8 15/30 Write block 430 len 20480 data: 6 6 6 6 6 16/30 Write block 490 len 8192 data: 4 4 17/30 Read block 532 len 20480 18/30 Flush 19/30 Write block 121 len 32768 data: 3 3 3 3 3 4 6 5 20/30 Flush 21/30 Write block 358 len 12288 data: 3 3 3 22/30 Write block 146 len 8192 data: 3 2 23/30 Write block 101 len 8192 data: 5 5 24/30 Write block 363 len 40960 data: 2 3 5 4 4 3 3 2 2 2 25/30 Read block 459 len 28672 26/30 Read block 281 len 16384 27/30 Write block 255 len 24576 data: 5 7 6 10 10 8 28/30 Write block 2 len 24576 data: 2 3 3 3 5 5 29/30 Write block 381 len 40960 data: 3 5 5 4 5 4 5 4 4 4 30/30 Read block 71 len 36864 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 0 29 Done 28 1 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1000)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:33.44520757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:33.445476383Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 32 32 32 T F T <--- 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 30 45 45 45 F F F 3 060-079 34 30 34 54 45 54 F F F <--- 4 080-099 34 28 34 54 43 54 F F F <--- 5 100-119 33 33 33 52 52 52 T F T <--- 6 120-139 34 32 34 55 50 55 F F F <--- 7 140-159 33 33 33 52 52 52 T F T <--- 8 160-179 34 33 34 54 52 54 F F F <--- 9 180-199 34 28 34 54 43 54 F F F <--- 10 200-219 34 32 34 54 51 54 F F F <--- 11 220-239 32 32 32 51 51 51 F F F 12 240-259 34 30 34 54 46 54 T F T <--- 13 260-279 34 32 34 54 50 54 T F T <--- 14 280-299 30 30 30 48 48 48 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 28 28 28 43 43 43 T F T <--- 18 360-379 31 31 31 49 49 49 T F T <--- 19 380-399 32 32 32 51 51 51 T F T <--- 20 400-419 32 32 32 51 51 51 F F F 21 420-439 34 32 34 54 50 54 F T F <--- 22 440-459 32 32 32 51 51 51 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 34 32 34 54 50 54 F F F <--- 25 500-519 34 18 34 54 30 54 F F F <--- 26 520-539 28 28 28 43 43 43 F F F 27 540-559 32 32 32 51 51 51 F F F 28 560-579 34 31 34 54 49 54 F F F <--- 29 580-599 34 30 34 54 46 54 F F F <--- Max gen: 34, Max flush: 55 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 30 30 30 45 45 45 F F F 3 060-079 34 34 34 54 54 54 F F F 4 080-099 34 34 34 54 54 54 F F F 5 100-119 35 35 35 56 56 56 F F F 6 120-139 34 34 34 55 55 55 F F F 7 140-159 35 35 35 56 56 56 F F F 8 160-179 34 34 34 54 54 54 F F F 9 180-199 34 34 34 54 54 54 F F F 10 200-219 34 34 34 54 54 54 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 35 35 35 56 56 56 F F F 13 260-279 35 35 35 56 56 56 F F F 14 280-299 30 30 30 48 48 48 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 35 35 35 56 56 56 F F F 19 380-399 35 35 35 56 56 56 F F F 20 400-419 32 32 32 51 51 51 F F F 21 420-439 34 34 34 54 54 54 F F F 22 440-459 32 32 32 51 51 51 F F F 23 460-479 15 15 15 27 27 27 F F F 24 480-499 34 34 34 54 54 54 F F F 25 500-519 34 34 34 54 54 54 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 32 32 32 51 51 51 F F F 28 560-579 34 34 34 54 54 54 F F F 29 580-599 34 34 34 54 54 54 F F F Max gen: 35, Max flush: 56 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.70956739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:37.717166207Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.717253918Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.724741723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725026907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725050497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725062067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725074067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Crucible 4bc4ddc3-e183-4bfb-bd87-34e626b6675e has session id: c10f5288-6ad7-42da-bef9-f1cc6098df6d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725393171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Upstairs opts: Upstairs UUID: 4bc4ddc3-e183-4bfb-bd87-34e626b6675e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725422491Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Crucible stats registered with UUID: 4bc4ddc3-e183-4bfb-bd87-34e626b6675e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725434271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:37.725444502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725634774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725665504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725676704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725768885Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725796146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"4bc4ddc3-e183-4bfb-bd87-34e626b6675e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725806586Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725815806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725825856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725838056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725848076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725858296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725868526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50730)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.725915857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52164)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726010298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49384)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726073409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726582755Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726602215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726798457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726892738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.726908358Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[0]R flush_numbers[0..12]: [56, 44, 45, 54, 54, 56, 55, 56, 54, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.72707639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[0]R generation[0..12]: [35, 29, 30, 34, 34, 35, 34, 35, 34, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.72708824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.72709975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[1]R flush_numbers[0..12]: [56, 44, 45, 54, 54, 56, 55, 56, 54, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727112291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[1]R generation[0..12]: [35, 29, 30, 34, 34, 35, 34, 35, 34, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727123231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727134551Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[2]R flush_numbers[0..12]: [56, 44, 45, 54, 54, 56, 55, 56, 54, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727145411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[2]R generation[0..12]: [35, 29, 30, 34, 34, 35, 34, 35, 34, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727156061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727167321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727177261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727187261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727196942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727206662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"":"downstairs","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727216922Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727226372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"0","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727236632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"1","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727246822Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"client":"2","":"downstairs","session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727257212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727267792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"4bc4ddc3-e183-4bfb-bd87-34e626b6675e is now active with session: c10f5288-6ad7-42da-bef9-f1cc6098df6d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727278053Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727287243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"session_id":"c10f5288-6ad7-42da-bef9-f1cc6098df6d"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727296573Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Activated sub_volume 4bc4ddc3-e183-4bfb-bd87-34e626b6675e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727304823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727313273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.727707887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} 01/30 Write block 173 len 32768 data: 4 3 3 3 4 4 4 6 02/30 Write block 296 len 40960 data: 3 3 3 4 4 4 4 6 6 5 03/30 Write block 70 len 8192 data: 3 3 04/30 Read block 148 len 20480 05/30 Write block 28 len 16384 data: 3 3 2 3 06/30 Read block 125 len 36864 07/30 Read block 312 len 28672 08/30 Write block 535 len 24576 data: 3 4 4 5 4 5 09/30 Write block 490 len 4096 data: 5 10/30 Write block 538 len 40960 data: 6 5 6 7 6 6 7 8 10 9 11/30 Write block 272 len 36864 data: 3 4 4 6 7 7 8 4 4 12/30 Read block 407 len 36864 13/30 Flush 14/30 Write block 386 len 12288 data: 5 6 5 15/30 Read block 159 len 12288 16/30 Read block 257 len 36864 17/30 Write block 102 len 12288 data: 6 6 6 18/30 Write block 51 len 36864 data: 4 3 4 4 5 5 4 5 7 19/30 Write block 430 len 4096 data: 7 20/30 Read block 129 len 36864 21/30 Write block 301 len 28672 data: 5 5 7 7 6 4 4 22/30 Read block 234 len 12288 23/30 Write block 474 len 40960 data: 4 3 3 3 3 2 3 3 3 5 24/30 Flush 25/30 Write block 550 len 12288 data: 4 4 4 26/30 Read block 265 len 16384 27/30 Write block 128 len 28672 data: 6 5 5 4 4 5 5 28/30 Read block 21 len 40960 29/30 Write block 255 len 32768 data: 6 8 7 11 11 9 9 9 30/30 Write block 375 len 24576 data: 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 9 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 2 2 29 Done 5 28 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:37.736805102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:37.737100995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 29 36 36 44 57 57 T F F <--- 2 040-059 30 36 36 45 58 58 F F F <--- 3 060-079 34 36 36 54 57 57 T F F <--- 4 080-099 34 34 34 54 54 54 F F F 5 100-119 35 36 36 56 58 58 F F F <--- 6 120-139 34 34 34 55 55 55 F T T <--- 7 140-159 35 35 35 56 56 56 F F F 8 160-179 34 36 36 54 57 57 T F F <--- 9 180-199 34 36 36 54 57 57 T F F <--- 10 200-219 34 34 34 54 54 54 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 35 35 35 56 56 56 F T T <--- 13 260-279 35 36 36 56 57 57 F T T <--- 14 280-299 30 36 36 48 57 57 T F F <--- 15 300-319 28 36 36 43 58 58 T F F <--- 16 320-339 28 28 28 43 43 43 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 35 35 35 56 56 56 F T T <--- 19 380-399 35 36 36 56 58 58 F T T <--- 20 400-419 32 32 32 51 51 51 F F F 21 420-439 34 36 36 54 58 58 F F F <--- 22 440-459 32 32 32 51 51 51 F F F 23 460-479 15 36 36 27 58 58 F F F <--- 24 480-499 34 36 36 54 58 58 F F F <--- 25 500-519 34 34 34 54 54 54 F F F 26 520-539 28 36 36 43 57 57 F F F <--- 27 540-559 32 36 36 51 57 57 F T T <--- 28 560-579 34 34 34 54 54 54 F F F 29 580-599 34 34 34 54 54 54 F F F Max gen: 36, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 36 36 36 57 57 57 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 36 36 36 57 57 57 F F F 4 080-099 34 34 34 54 54 54 F F F 5 100-119 36 36 36 58 58 58 F F F 6 120-139 37 37 37 59 59 59 F F F 7 140-159 35 35 35 56 56 56 F F F 8 160-179 36 36 36 57 57 57 F F F 9 180-199 36 36 36 57 57 57 F F F 10 200-219 34 34 34 54 54 54 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 37 37 37 59 59 59 F F F 13 260-279 37 37 37 59 59 59 F F F 14 280-299 36 36 36 57 57 57 F F F 15 300-319 36 36 36 58 58 58 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 32 32 32 51 51 51 F F F 21 420-439 36 36 36 58 58 58 F F F 22 440-459 32 32 32 51 51 51 F F F 23 460-479 36 36 36 58 58 58 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 34 34 34 54 54 54 F F F 26 520-539 36 36 36 57 57 57 F F F 27 540-559 37 37 37 59 59 59 F F F 28 560-579 34 34 34 54 54 54 F F F 29 580-599 34 34 34 54 54 54 F F F Max gen: 37, Max flush: 59 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.003224355Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012213638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012536692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012562792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012571782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012583822Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Crucible 5f26b6b6-2d64-40d1-82f4-895f604e0d52 has session id: 4c45d76c-7fa2-4714-9a6d-3f22f5145d90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012941466Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Upstairs opts: Upstairs UUID: 5f26b6b6-2d64-40d1-82f4-895f604e0d52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.012987577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Crucible stats registered with UUID: 5f26b6b6-2d64-40d1-82f4-895f604e0d52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013000417Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:42.013010157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013194809Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013210209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013415452Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013433372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"5f26b6b6-2d64-40d1-82f4-895f604e0d52 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013452102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013461412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013471392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013481373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013491393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013501943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013512523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:48231)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013641074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:56912)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013667835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:57029)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.013722455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014378123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014399613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014458994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014700607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014740127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 58, 57, 54, 58, 59, 56, 57, 57, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014788428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[0]R generation[0..12]: [35, 36, 36, 36, 34, 36, 37, 35, 36, 36, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014799798Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014811428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 58, 57, 54, 58, 59, 56, 57, 57, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014822288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[1]R generation[0..12]: [35, 36, 36, 36, 34, 36, 37, 35, 36, 36, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014833018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014844458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 58, 57, 54, 58, 59, 56, 57, 57, 54, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014855318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[2]R generation[0..12]: [35, 36, 36, 36, 34, 36, 37, 35, 36, 36, 34, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014866258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014879169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014889179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014899379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014909239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014919019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"":"downstairs","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014929119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014938229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"0","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014948249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"1","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.014958369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"client":"2","":"downstairs","session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01496858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01497951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"5f26b6b6-2d64-40d1-82f4-895f604e0d52 is now active with session: 4c45d76c-7fa2-4714-9a6d-3f22f5145d90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01499002Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01499942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"session_id":"4c45d76c-7fa2-4714-9a6d-3f22f5145d90"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01500891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Activated sub_volume 5f26b6b6-2d64-40d1-82f4-895f604e0d52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.01501742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.0150261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.015476495Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} 01/30 Write block 200 len 28672 data: 2 3 3 3 5 4 3 02/30 Read block 385 len 24576 03/30 Write block 154 len 4096 data: 2 04/30 Write block 510 len 16384 data: 4 4 3 3 05/30 Read block 389 len 16384 06/30 Read block 235 len 40960 07/30 Read block 533 len 4096 08/30 Write block 245 len 36864 data: 3 3 3 4 3 4 4 4 5 09/30 Write block 246 len 24576 data: 4 4 5 4 5 5 10/30 Read block 484 len 16384 11/30 Read block 474 len 36864 12/30 Write block 262 len 20480 data: 10 9 7 7 4 13/30 Write block 20 len 12288 data: 2 2 2 14/30 Write block 70 len 20480 data: 4 4 4 4 4 15/30 Write block 295 len 40960 data: 2 4 4 4 5 5 6 6 8 8 16/30 Write block 38 len 8192 data: 2 2 17/30 Write block 90 len 32768 data: 4 4 5 5 5 6 7 7 18/30 Flush 19/30 Write block 565 len 16384 data: 6 6 6 7 20/30 Write block 512 len 8192 data: 4 4 21/30 Write block 578 len 4096 data: 11 22/30 Write block 413 len 8192 data: 4 4 23/30 Read block 567 len 24576 24/30 Write block 516 len 24576 data: 3 3 3 2 2 2 25/30 Read block 134 len 20480 26/30 Read block 526 len 16384 27/30 Write block 442 len 4096 data: 5 28/30 Write block 320 len 36864 data: 4 4 4 4 2 2 3 3 3 29/30 Read block 21 len 8192 30/30 Read block 65 len 40960 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:42.023285735Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:42.023573328Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 38 38 36 60 60 57 F F F <--- 2 040-059 36 36 36 58 58 58 F F F 3 060-079 38 38 36 60 60 57 F F F <--- 4 080-099 38 38 34 60 60 54 F F F <--- 5 100-119 36 36 36 58 58 58 F F F 6 120-139 37 37 37 59 59 59 F F F 7 140-159 38 38 35 60 60 56 F F F <--- 8 160-179 36 36 36 57 57 57 F F F 9 180-199 36 36 36 57 57 57 F F F 10 200-219 38 38 34 60 60 54 F F T <--- 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 37 60 60 59 F F F <--- 13 260-279 38 38 37 60 60 59 F F F <--- 14 280-299 38 38 36 60 60 57 F F F <--- 15 300-319 38 38 36 60 60 58 F F F <--- 16 320-339 28 28 28 43 43 43 T T F <--- 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 32 32 32 51 51 51 T T F <--- 21 420-439 36 36 36 58 58 58 F F F 22 440-459 32 32 32 51 51 51 T T F <--- 23 460-479 36 36 36 58 58 58 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 38 38 34 60 60 54 T T F <--- 26 520-539 36 36 36 57 57 57 T T F <--- 27 540-559 37 37 37 59 59 59 F F F 28 560-579 34 34 34 54 54 54 T T F <--- 29 580-599 34 34 34 54 54 54 F F F Max gen: 38, Max flush: 60 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 38 38 38 60 60 60 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 38 38 38 60 60 60 F F F 4 080-099 38 38 38 60 60 60 F F F 5 100-119 36 36 36 58 58 58 F F F 6 120-139 37 37 37 59 59 59 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 36 36 36 57 57 57 F F F 9 180-199 36 36 36 57 57 57 F F F 10 200-219 38 38 38 60 60 60 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 38 38 38 60 60 60 F F F 14 280-299 38 38 38 60 60 60 F F F 15 300-319 38 38 38 60 60 60 F F F 16 320-339 39 39 39 61 61 61 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 37 37 37 59 59 59 F F F 20 400-419 39 39 39 61 61 61 F F F 21 420-439 36 36 36 58 58 58 F F F 22 440-459 39 39 39 61 61 61 F F F 23 460-479 36 36 36 58 58 58 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 39 39 39 61 61 61 F F F 26 520-539 39 39 39 61 61 61 F F F 27 540-559 37 37 37 59 59 59 F F F 28 560-579 39 39 39 61 61 61 F F F 29 580-599 34 34 34 54 54 54 F F F Max gen: 39, Max flush: 61 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.272581973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281041689Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281335303Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281358903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281368083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281380013Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Crucible 3816217f-e821-4ad3-8ca1-f45cf7e89b1e has session id: 5ab0bef3-0849-414c-ae6e-3418fb61a5ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281763008Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Upstairs opts: Upstairs UUID: 3816217f-e821-4ad3-8ca1-f45cf7e89b1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281804628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Crucible stats registered with UUID: 3816217f-e821-4ad3-8ca1-f45cf7e89b1e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.281815828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:46.281825348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282005791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282021661Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282032181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282199373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282216993Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"3816217f-e821-4ad3-8ca1-f45cf7e89b1e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282233053Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282242223Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282252243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282262243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282272114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282282384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282292604Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:52399)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282360215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:50000)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282371415Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:42117)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.282381925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283074333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283116523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283154204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283409687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283425257Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[0]R flush_numbers[0..12]: [56, 60, 58, 60, 60, 58, 59, 60, 57, 57, 60, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283457157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[0]R generation[0..12]: [35, 38, 36, 38, 38, 36, 37, 38, 36, 36, 38, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283468567Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283480097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[1]R flush_numbers[0..12]: [56, 60, 58, 60, 60, 58, 59, 60, 57, 57, 60, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283490687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[1]R generation[0..12]: [35, 38, 36, 38, 38, 36, 37, 38, 36, 36, 38, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283501298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283512488Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[2]R flush_numbers[0..12]: [56, 60, 58, 60, 60, 58, 59, 60, 57, 57, 60, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283523208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[2]R generation[0..12]: [35, 38, 36, 38, 38, 36, 37, 38, 36, 36, 38, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283533858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283545088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283554948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283564958Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283574618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283584339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"":"downstairs","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283594559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283605069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"0","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283615099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"1","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283625149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"client":"2","":"downstairs","session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283635289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283644699Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"3816217f-e821-4ad3-8ca1-f45cf7e89b1e is now active with session: 5ab0bef3-0849-414c-ae6e-3418fb61a5ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283655139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.283664279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"session_id":"5ab0bef3-0849-414c-ae6e-3418fb61a5ea"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.28367369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Activated sub_volume 3816217f-e821-4ad3-8ca1-f45cf7e89b1e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.28368283Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.28369145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.284066254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} 01/30 Flush 02/30 Write block 186 len 4096 data: 5 03/30 Read block 49 len 8192 04/30 Read block 11 len 36864 05/30 Read block 132 len 36864 06/30 Write block 163 len 32768 data: 5 5 5 5 5 5 4 4 07/30 Write block 263 len 16384 data: 10 8 8 5 08/30 Read block 127 len 40960 09/30 Write block 185 len 4096 data: 5 10/30 Read block 41 len 20480 11/30 Write block 208 len 4096 data: 4 12/30 Read block 340 len 24576 13/30 Write block 273 len 12288 data: 5 5 7 14/30 Write block 181 len 32768 data: 5 4 4 4 6 6 5 4 15/30 Read block 183 len 24576 16/30 Flush 17/30 Write block 97 len 40960 data: 8 5 4 5 6 7 7 7 9 9 18/30 Read block 383 len 4096 19/30 Read block 232 len 32768 20/30 Write block 557 len 12288 data: 4 6 7 21/30 Flush 22/30 Write block 322 len 28672 data: 5 5 3 3 4 4 4 23/30 Read block 300 len 40960 24/30 Write block 383 len 24576 data: 6 5 6 6 7 6 25/30 Read block 7 len 24576 26/30 Read block 289 len 20480 27/30 Read block 380 len 8192 28/30 Write block 556 len 32768 data: 3 5 7 8 6 6 7 7 29/30 Read block 127 len 28672 30/30 Write block 458 len 28672 data: 4 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 28 28 4 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:46.292776234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:46.293024457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 38 38 38 60 60 60 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 38 38 38 60 60 60 F F F 4 080-099 40 40 38 64 64 60 F F F <--- 5 100-119 40 40 36 64 64 58 F F F <--- 6 120-139 37 37 37 59 59 59 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 40 40 36 63 63 57 F F F <--- 9 180-199 40 40 36 63 63 57 F F T <--- 10 200-219 40 40 38 63 63 60 F F F <--- 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 40 40 38 63 63 60 F F F <--- 14 280-299 38 38 38 60 60 60 F F F 15 300-319 38 38 38 60 60 60 F F F 16 320-339 39 39 39 61 61 61 T T F <--- 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 37 37 37 59 59 59 T T F <--- 20 400-419 39 39 39 61 61 61 F F F 21 420-439 36 36 36 58 58 58 F F F 22 440-459 39 39 39 61 61 61 T T F <--- 23 460-479 36 36 36 58 58 58 T T F <--- 24 480-499 36 36 36 58 58 58 F F F 25 500-519 39 39 39 61 61 61 F F F 26 520-539 39 39 39 61 61 61 F F F 27 540-559 40 40 37 64 64 59 T T F <--- 28 560-579 39 39 39 61 61 61 T T F <--- 29 580-599 34 34 34 54 54 54 F F F Max gen: 40, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 38 38 38 60 60 60 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 38 38 38 60 60 60 F F F 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 37 37 37 59 59 59 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 40 40 40 63 63 63 F F F 9 180-199 40 40 40 63 63 63 F F F 10 200-219 40 40 40 63 63 63 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 40 40 40 63 63 63 F F F 14 280-299 38 38 38 60 60 60 F F F 15 300-319 38 38 38 60 60 60 F F F 16 320-339 41 41 41 65 65 65 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 41 41 41 65 65 65 F F F 20 400-419 39 39 39 61 61 61 F F F 21 420-439 36 36 36 58 58 58 F F F 22 440-459 41 41 41 65 65 65 F F F 23 460-479 41 41 41 65 65 65 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 39 39 39 61 61 61 F F F 26 520-539 39 39 39 61 61 61 F F F 27 540-559 41 41 41 65 65 65 F F F 28 560-579 41 41 41 65 65 65 F F F 29 580-599 34 34 34 54 54 54 F F F Max gen: 41, Max flush: 65 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.535222473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.543800271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544121935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544151145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544160335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544172236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Crucible 90ced15b-1317-41ed-9992-7fc94bc288b7 has session id: c781ddbf-9b0b-47cf-bb21-7206e550bd07","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544508249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Upstairs opts: Upstairs UUID: 90ced15b-1317-41ed-9992-7fc94bc288b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54453817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Crucible stats registered with UUID: 90ced15b-1317-41ed-9992-7fc94bc288b7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54455034Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:50.54456099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544741732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544758052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544768782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544932044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544951034Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"90ced15b-1317-41ed-9992-7fc94bc288b7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544964635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544974125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544984165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.544994235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545014635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545024975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545037095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"ds_connection connected from Ok(127.0.0.1:44787)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545064866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"ds_connection connected from Ok(127.0.0.1:33592)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545076256Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"ds_connection connected from Ok(127.0.0.1:62781)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545086906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545760054Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545843065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.545864845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546087217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546111638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[0]R flush_numbers[0..12]: [56, 60, 58, 60, 64, 64, 59, 60, 63, 63, 63, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546153898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[0]R generation[0..12]: [35, 38, 36, 38, 40, 40, 37, 38, 40, 40, 40, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546166038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546177628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[1]R flush_numbers[0..12]: [56, 60, 58, 60, 64, 64, 59, 60, 63, 63, 63, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546194349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[1]R generation[0..12]: [35, 38, 36, 38, 40, 40, 37, 38, 40, 40, 40, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546205149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546219059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[2]R flush_numbers[0..12]: [56, 60, 58, 60, 64, 64, 59, 60, 63, 63, 63, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546229919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[2]R generation[0..12]: [35, 38, 36, 38, 40, 40, 37, 38, 40, 40, 40, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546240669Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546251989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546261869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54627196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54628238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54629218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"":"downstairs","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54630231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54631166Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"0","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54632173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"1","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54633185Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"client":"2","":"downstairs","session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54634211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.54635165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"90ced15b-1317-41ed-9992-7fc94bc288b7 is now active with session: c781ddbf-9b0b-47cf-bb21-7206e550bd07","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546361961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546371251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"session_id":"c781ddbf-9b0b-47cf-bb21-7206e550bd07"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546381861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Activated sub_volume 90ced15b-1317-41ed-9992-7fc94bc288b7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546390531Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546399021Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.546772245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} 01/30 Write block 405 len 8192 data: 6 5 02/30 Read block 517 len 40960 03/30 Read block 191 len 40960 04/30 Write block 191 len 24576 data: 6 7 7 7 6 5 05/30 Read block 129 len 20480 06/30 Write block 450 len 16384 data: 4 3 2 3 07/30 Read block 8 len 20480 08/30 Read block 548 len 20480 09/30 Write block 575 len 32768 data: 10 11 9 12 12 11 10 9 10/30 Read block 240 len 24576 11/30 Read block 254 len 36864 12/30 Flush 13/30 Write block 273 len 28672 data: 6 6 8 8 8 9 5 14/30 Read block 306 len 20480 15/30 Write block 390 len 20480 data: 5 5 5 5 5 16/30 Flush 17/30 Write block 67 len 24576 data: 4 3 3 5 5 5 18/30 Write block 33 len 16384 data: 3 2 2 2 19/30 Read block 217 len 12288 20/30 Write block 478 len 8192 data: 4 3 21/30 Write block 128 len 32768 data: 7 6 6 5 5 6 6 3 22/30 Read block 231 len 36864 23/30 Read block 239 len 16384 24/30 Write block 283 len 12288 data: 3 4 4 25/30 Read block 420 len 24576 26/30 Read block 487 len 32768 27/30 Read block 138 len 24576 28/30 Write block 263 len 28672 data: 11 9 9 6 5 4 4 29/30 Write block 202 len 16384 data: 4 4 6 5 30/30 Write block 509 len 8192 data: 4 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 25 31 Done 27 27 5 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:50.555733628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:50.556034561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 38 38 38 60 60 60 T T F <--- 2 040-059 36 36 36 58 58 58 F F F 3 060-079 38 38 38 60 60 60 T T F <--- 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 37 37 37 59 59 59 T T F <--- 7 140-159 38 38 38 60 60 60 F F F 8 160-179 40 40 40 63 63 63 F F F 9 180-199 42 42 40 66 66 63 F F T <--- 10 200-219 40 40 40 63 63 63 T T F <--- 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 42 42 40 67 67 63 T T F <--- 14 280-299 38 38 38 60 60 60 T T F <--- 15 300-319 38 38 38 60 60 60 F F F 16 320-339 41 41 41 65 65 65 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 42 42 41 67 67 65 F F F <--- 20 400-419 42 42 39 66 66 61 F F T <--- 21 420-439 36 36 36 58 58 58 F F F 22 440-459 42 42 41 66 66 65 F F F <--- 23 460-479 41 41 41 65 65 65 T T F <--- 24 480-499 36 36 36 58 58 58 F F F 25 500-519 39 39 39 61 61 61 T T F <--- 26 520-539 39 39 39 61 61 61 F F F 27 540-559 41 41 41 65 65 65 F F F 28 560-579 42 42 41 66 66 65 F F F <--- 29 580-599 42 42 34 66 66 54 F F F <--- Max gen: 42, Max flush: 67 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 43 43 43 68 68 68 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 43 43 43 68 68 68 F F F 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 43 43 43 68 68 68 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 40 40 40 63 63 63 F F F 9 180-199 42 42 42 66 66 66 F F F 10 200-219 43 43 43 68 68 68 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 43 43 43 68 68 68 F F F 14 280-299 43 43 43 68 68 68 F F F 15 300-319 38 38 38 60 60 60 F F F 16 320-339 41 41 41 65 65 65 F F F 17 340-359 35 35 35 56 56 56 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 42 42 42 67 67 67 F F F 20 400-419 42 42 42 66 66 66 F F F 21 420-439 36 36 36 58 58 58 F F F 22 440-459 42 42 42 66 66 66 F F F 23 460-479 43 43 43 68 68 68 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 43 43 43 68 68 68 F F F 26 520-539 39 39 39 61 61 61 F F F 27 540-559 41 41 41 65 65 65 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 42 42 42 66 66 66 F F F Max gen: 43, Max flush: 68 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.804663291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813164609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813454892Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813479202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813488292Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813500052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Crucible bfab6486-af6d-4209-ae0c-cd8f9e3b9802 has session id: 2dd7c5a3-49ad-4978-b082-14c43872229f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813882847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Upstairs opts: Upstairs UUID: bfab6486-af6d-4209-ae0c-cd8f9e3b9802, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813912547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Crucible stats registered with UUID: bfab6486-af6d-4209-ae0c-cd8f9e3b9802","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.813923977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:54.813933817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814112029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.81413841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.81414992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814341752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814366352Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"bfab6486-af6d-4209-ae0c-cd8f9e3b9802 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814379343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814388483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814398523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814408543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814418343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814428513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814438803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64653)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814482584Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53593)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814564115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39670)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.814625715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815215642Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815229392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815243412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815540466Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815550506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[0]R flush_numbers[0..12]: [56, 68, 58, 68, 64, 64, 68, 60, 63, 66, 68, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815588136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[0]R generation[0..12]: [35, 43, 36, 43, 40, 40, 43, 38, 40, 42, 43, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815599527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815610817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[1]R flush_numbers[0..12]: [56, 68, 58, 68, 64, 64, 68, 60, 63, 66, 68, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815621697Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[1]R generation[0..12]: [35, 43, 36, 43, 40, 40, 43, 38, 40, 42, 43, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815632367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815643687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[2]R flush_numbers[0..12]: [56, 68, 58, 68, 64, 64, 68, 60, 63, 66, 68, 51]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815654367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[2]R generation[0..12]: [35, 43, 36, 43, 40, 40, 43, 38, 40, 42, 43, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815664897Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815676167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815685838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815695898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815706968Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815716628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"":"downstairs","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815726608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815735588Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"0","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815745658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"1","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815755468Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"client":"2","":"downstairs","session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815765558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815775069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"bfab6486-af6d-4209-ae0c-cd8f9e3b9802 is now active with session: 2dd7c5a3-49ad-4978-b082-14c43872229f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815785299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815794399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"session_id":"2dd7c5a3-49ad-4978-b082-14c43872229f"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815803839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Activated sub_volume bfab6486-af6d-4209-ae0c-cd8f9e3b9802","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815812189Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.815820589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.816205163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} 01/30 Write block 337 len 36864 data: 3 3 4 4 5 5 4 4 4 02/30 Write block 294 len 16384 data: 2 3 5 5 03/30 Read block 71 len 12288 04/30 Read block 209 len 36864 05/30 Write block 429 len 32768 data: 4 8 7 7 7 7 7 8 06/30 Write block 65 len 16384 data: 5 5 5 4 07/30 Read block 464 len 28672 08/30 Write block 225 len 20480 data: 3 3 4 4 4 09/30 Flush 10/30 Write block 196 len 4096 data: 6 11/30 Write block 175 len 8192 data: 4 4 12/30 Read block 510 len 28672 13/30 Read block 356 len 28672 14/30 Read block 226 len 32768 15/30 Write block 516 len 28672 data: 4 4 4 3 3 3 2 16/30 Read block 173 len 12288 17/30 Write block 174 len 12288 data: 4 5 5 18/30 Write block 132 len 32768 data: 6 7 7 4 3 3 4 3 19/30 Read block 447 len 20480 20/30 Read block 370 len 8192 21/30 Read block 355 len 36864 22/30 Read block 412 len 20480 23/30 Read block 209 len 8192 24/30 Write block 201 len 8192 data: 4 5 25/30 Read block 309 len 12288 26/30 Read block 198 len 32768 27/30 Write block 301 len 40960 data: 7 7 9 9 7 5 5 4 4 4 28/30 Write block 183 len 16384 data: 5 5 7 7 29/30 Write block 411 len 24576 data: 4 4 5 5 4 4 30/30 Write block 265 len 32768 data: 10 7 6 5 5 4 4 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 26 34 Done 26 26 4 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:54.824256485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:54.824520878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 43 43 43 68 68 68 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 44 44 43 69 69 68 F F F <--- 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 43 43 43 68 68 68 T T F <--- 7 140-159 38 38 38 60 60 60 F F F 8 160-179 40 40 40 63 63 63 T T F <--- 9 180-199 42 42 42 66 66 66 T T F <--- 10 200-219 43 43 43 68 68 68 T T F <--- 11 220-239 44 44 32 69 69 51 F F F <--- 12 240-259 38 38 38 60 60 60 F F F 13 260-279 43 43 43 68 68 68 T T F <--- 14 280-299 44 44 43 69 69 68 F F T <--- 15 300-319 38 38 38 60 60 60 T T F <--- 16 320-339 44 44 41 69 69 65 F F T <--- 17 340-359 44 44 35 69 69 56 F F T <--- 18 360-379 37 37 37 59 59 59 F F F 19 380-399 42 42 42 67 67 67 F F F 20 400-419 42 42 42 66 66 66 T T F <--- 21 420-439 44 44 36 69 69 58 F F F <--- 22 440-459 42 42 42 66 66 66 F F F 23 460-479 43 43 43 68 68 68 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 43 43 43 68 68 68 T T F <--- 26 520-539 39 39 39 61 61 61 T T F <--- 27 540-559 41 41 41 65 65 65 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 42 42 42 66 66 66 F F F Max gen: 44, Max flush: 69 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 56 56 56 F F F 1 020-039 43 43 43 68 68 68 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 44 44 44 69 69 69 F F F 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 45 45 45 70 70 70 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 45 45 45 70 70 70 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 45 45 45 70 70 70 F F F 11 220-239 44 44 44 69 69 69 F F F 12 240-259 38 38 38 60 60 60 F F F 13 260-279 45 45 45 70 70 70 F F F 14 280-299 44 44 44 69 69 69 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 44 44 44 69 69 69 F F F 17 340-359 44 44 44 69 69 69 F F F 18 360-379 37 37 37 59 59 59 F F F 19 380-399 42 42 42 67 67 67 F F F 20 400-419 45 45 45 70 70 70 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 42 42 42 66 66 66 F F F 23 460-479 43 43 43 68 68 68 F F F 24 480-499 36 36 36 58 58 58 F F F 25 500-519 45 45 45 70 70 70 F F F 26 520-539 45 45 45 70 70 70 F F F 27 540-559 41 41 41 65 65 65 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 42 42 42 66 66 66 F F F Max gen: 45, Max flush: 70 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.085306367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.093847765Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094151728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094181318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094192549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094204699Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Crucible 0884a6a0-01c1-43ff-bb38-86c47fb1e88c has session id: 6b706792-a23a-42df-838b-c49300b1c5b1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094577333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Upstairs opts: Upstairs UUID: 0884a6a0-01c1-43ff-bb38-86c47fb1e88c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094632834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Crucible stats registered with UUID: 0884a6a0-01c1-43ff-bb38-86c47fb1e88c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094644534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:59.094654134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094823846Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094841426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.094852006Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095024548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095051088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"0884a6a0-01c1-43ff-bb38-86c47fb1e88c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095063149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095072469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095082519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095094249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095104359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095114629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095124979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42655)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.09517666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35737)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.0951891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34732)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095245981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095958609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095983099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.095995019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096281003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096290963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[0]R flush_numbers[0..12]: [56, 68, 58, 69, 64, 64, 70, 60, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096304083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[0]R generation[0..12]: [35, 43, 36, 44, 40, 40, 45, 38, 45, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096315003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096326673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[1]R flush_numbers[0..12]: [56, 68, 58, 69, 64, 64, 70, 60, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096339143Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[1]R generation[0..12]: [35, 43, 36, 44, 40, 40, 45, 38, 45, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096349923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096361213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[2]R flush_numbers[0..12]: [56, 68, 58, 69, 64, 64, 70, 60, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096371884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[2]R generation[0..12]: [35, 43, 36, 44, 40, 40, 45, 38, 45, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096382624Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096393994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096403834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096413924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096423594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096433484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"":"downstairs","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096443644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096452974Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"0","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096463015Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"1","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096473145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"client":"2","":"downstairs","session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096484845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096494305Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"0884a6a0-01c1-43ff-bb38-86c47fb1e88c is now active with session: 6b706792-a23a-42df-838b-c49300b1c5b1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096504825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096513965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"session_id":"6b706792-a23a-42df-838b-c49300b1c5b1"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096523465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Activated sub_volume 0884a6a0-01c1-43ff-bb38-86c47fb1e88c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096531835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.096540455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.09697489Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} 01/30 Write block 411 len 36864 data: 5 5 6 6 5 5 4 3 3 02/30 Write block 160 len 40960 data: 3 4 4 6 6 6 6 6 6 5 03/30 Write block 596 len 8192 data: 2 2 04/30 Read block 528 len 32768 05/30 Read block 204 len 24576 06/30 Write block 249 len 40960 data: 5 6 6 5 6 4 7 9 8 12 07/30 Write block 456 len 28672 data: 4 4 5 4 4 4 4 08/30 Write block 209 len 40960 data: 4 4 6 4 2 2 2 3 4 2 09/30 Write block 241 len 40960 data: 4 6 3 3 4 5 5 6 6 7 10/30 Read block 463 len 4096 11/30 Write block 583 len 8192 data: 6 8 12/30 Write block 466 len 32768 data: 2 2 2 3 4 4 4 4 13/30 Flush 14/30 Write block 405 len 16384 data: 7 6 4 3 15/30 Read block 318 len 20480 16/30 Write block 534 len 32768 data: 4 4 5 5 7 6 7 8 17/30 Flush 18/30 Write block 395 len 20480 data: 5 4 5 7 6 19/30 Read block 101 len 36864 20/30 Read block 306 len 16384 21/30 Write block 0 len 8192 data: 2 2 22/30 Read block 196 len 8192 23/30 Write block 365 len 8192 data: 6 5 24/30 Flush 25/30 Write block 498 len 24576 data: 4 4 5 5 4 3 26/30 Write block 443 len 8192 data: 3 3 27/30 Write block 116 len 28672 data: 3 3 3 3 2 4 4 28/30 Write block 291 len 12288 data: 2 3 3 29/30 Read block 573 len 20480 30/30 Read block 256 len 36864 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 0 28 Done 29 3 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:21:59.105024113Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:21:59.105300176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 35 46 73 56 73 F F F <--- 1 020-039 43 43 43 68 68 68 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 44 44 44 69 69 69 F F F 4 080-099 40 40 40 64 64 64 F F F 5 100-119 40 40 40 64 64 64 T F T <--- 6 120-139 45 45 45 70 70 70 T F T <--- 7 140-159 38 38 38 60 60 60 F F F 8 160-179 46 45 46 71 70 71 F T F <--- 9 180-199 45 45 45 70 70 70 F F F 10 200-219 46 45 46 71 70 71 F F F <--- 11 220-239 44 44 44 69 69 69 F F F 12 240-259 46 38 46 71 60 71 F F F <--- 13 260-279 45 45 45 70 70 70 F F F 14 280-299 44 44 44 69 69 69 T F T <--- 15 300-319 45 45 45 70 70 70 F F F 16 320-339 44 44 44 69 69 69 F F F 17 340-359 44 44 44 69 69 69 F F F 18 360-379 46 37 46 73 59 73 F F F <--- 19 380-399 46 42 46 73 67 73 F F F <--- 20 400-419 46 45 46 72 70 72 F T F <--- 21 420-439 44 44 44 69 69 69 F F F 22 440-459 46 42 46 71 66 71 T F T <--- 23 460-479 46 43 46 71 68 71 F F F <--- 24 480-499 36 36 36 58 58 58 T F T <--- 25 500-519 45 45 45 70 70 70 T F T <--- 26 520-539 46 45 46 72 70 72 F F F <--- 27 540-559 46 41 46 72 65 72 F F F <--- 28 560-579 42 42 42 66 66 66 F F F 29 580-599 46 42 46 71 66 71 F T F <--- Max gen: 46, Max flush: 73 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 73 73 73 F F F 1 020-039 43 43 43 68 68 68 F F F 2 040-059 36 36 36 58 58 58 F F F 3 060-079 44 44 44 69 69 69 F F F 4 080-099 40 40 40 64 64 64 F F F 5 100-119 47 47 47 74 74 74 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 46 46 46 71 71 71 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 46 46 46 71 71 71 F F F 11 220-239 44 44 44 69 69 69 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 45 45 45 70 70 70 F F F 14 280-299 47 47 47 74 74 74 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 44 44 44 69 69 69 F F F 17 340-359 44 44 44 69 69 69 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 46 46 46 71 71 71 F F F 24 480-499 47 47 47 74 74 74 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 72 72 72 F F F 27 540-559 46 46 46 72 72 72 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 46 46 46 71 71 71 F F F Max gen: 47, Max flush: 74 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.358004302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:03.365477098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.365542749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.372806562Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373067155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373090985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373099825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373111645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Crucible d80e1e28-5d71-4176-a0c4-32ecf7c6f53b has session id: ccb7d9a1-df50-46ff-ae88-27488cc16f8a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373451259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Upstairs opts: Upstairs UUID: d80e1e28-5d71-4176-a0c4-32ecf7c6f53b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37348343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Crucible stats registered with UUID: d80e1e28-5d71-4176-a0c4-32ecf7c6f53b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37349559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:03.37350776Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373696342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373727422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373738903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373874714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373891874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"d80e1e28-5d71-4176-a0c4-32ecf7c6f53b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373903714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373912965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373923095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373933115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373942985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373953525Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.373965825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44315)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374073076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44034)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374093107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63976)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374104287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374822565Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374848335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.374877226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375138429Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375174759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[0]R flush_numbers[0..12]: [73, 68, 58, 69, 64, 74, 74, 60, 71, 70, 71, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.3752286Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[0]R generation[0..12]: [46, 43, 36, 44, 40, 47, 47, 38, 46, 45, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37524114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37525301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[1]R flush_numbers[0..12]: [73, 68, 58, 69, 64, 74, 74, 60, 71, 70, 71, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37526379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[1]R generation[0..12]: [46, 43, 36, 44, 40, 47, 47, 38, 46, 45, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37527453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37528589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[2]R flush_numbers[0..12]: [73, 68, 58, 69, 64, 74, 74, 60, 71, 70, 71, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.37529896Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[2]R generation[0..12]: [46, 43, 36, 44, 40, 47, 47, 38, 46, 45, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375309921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375321321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375331101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375341101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375350961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375360681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"":"downstairs","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375370941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375380181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"0","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375390191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"1","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375400262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"client":"2","":"downstairs","session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375410632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375420172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"d80e1e28-5d71-4176-a0c4-32ecf7c6f53b is now active with session: ccb7d9a1-df50-46ff-ae88-27488cc16f8a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375430442Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375439622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"session_id":"ccb7d9a1-df50-46ff-ae88-27488cc16f8a"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375450472Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Activated sub_volume d80e1e28-5d71-4176-a0c4-32ecf7c6f53b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375458972Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375467382Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.375881377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} 01/30 Write block 277 len 12288 data: 9 10 6 02/30 Read block 459 len 20480 03/30 Read block 381 len 8192 04/30 Read block 237 len 32768 05/30 Read block 68 len 12288 06/30 Read block 582 len 16384 07/30 Read block 235 len 16384 08/30 Flush 09/30 Write block 583 len 8192 data: 7 9 10/30 Read block 4 len 24576 11/30 Read block 213 len 32768 12/30 Read block 347 len 4096 13/30 Write block 80 len 12288 data: 5 6 7 14/30 Read block 467 len 4096 15/30 Read block 185 len 32768 16/30 Flush 17/30 Write block 50 len 28672 data: 3 5 4 5 5 6 6 18/30 Flush 19/30 Write block 19 len 12288 data: 3 3 3 20/30 Write block 17 len 32768 data: 4 3 4 4 4 3 2 2 21/30 Read block 119 len 24576 22/30 Read block 386 len 28672 23/30 Read block 407 len 36864 24/30 Read block 206 len 16384 25/30 Write block 441 len 40960 data: 5 6 4 4 3 3 3 2 2 5 26/30 Read block 284 len 8192 27/30 Read block 439 len 8192 28/30 Write block 202 len 8192 data: 6 5 29/30 Read block 242 len 8192 30/30 Read block 220 len 20480 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:03.384415415Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:03.384685098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 73 73 73 F T T <--- 1 020-039 43 43 43 68 68 68 F T T <--- 2 040-059 36 48 48 58 77 77 F F F <--- 3 060-079 44 44 44 69 69 69 F F F 4 080-099 40 48 48 64 76 76 F F F <--- 5 100-119 47 47 47 74 74 74 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 46 46 46 71 71 71 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 46 46 46 71 71 71 F T T <--- 11 220-239 44 44 44 69 69 69 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 45 48 48 70 75 75 F F F <--- 14 280-299 47 47 47 74 74 74 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 44 44 44 69 69 69 F F F 17 340-359 44 44 44 69 69 69 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 47 47 47 74 74 74 F T T <--- 23 460-479 46 46 46 71 71 71 F F F 24 480-499 47 47 47 74 74 74 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 72 72 72 F F F 27 540-559 46 46 46 72 72 72 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 46 48 48 71 76 76 F F F <--- Max gen: 48, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 44 44 44 69 69 69 F F F 4 080-099 48 48 48 76 76 76 F F F 5 100-119 47 47 47 74 74 74 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 38 38 38 60 60 60 F F F 8 160-179 46 46 46 71 71 71 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 49 49 49 78 78 78 F F F 11 220-239 44 44 44 69 69 69 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 48 48 48 75 75 75 F F F 14 280-299 47 47 47 74 74 74 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 44 44 44 69 69 69 F F F 17 340-359 44 44 44 69 69 69 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 49 49 49 78 78 78 F F F 23 460-479 46 46 46 71 71 71 F F F 24 480-499 47 47 47 74 74 74 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 72 72 72 F F F 27 540-559 46 46 46 72 72 72 F F F 28 560-579 42 42 42 66 66 66 F F F 29 580-599 48 48 48 76 76 76 F F F Max gen: 49, Max flush: 78 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.632465728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641253909Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641549312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641573432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641582533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641594603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Crucible e2397831-0436-430d-924d-0020efe3ddcc has session id: 47a9559a-13e7-449d-aa71-83e3f8e1932b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.641951717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Upstairs opts: Upstairs UUID: e2397831-0436-430d-924d-0020efe3ddcc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642026248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Crucible stats registered with UUID: e2397831-0436-430d-924d-0020efe3ddcc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642037388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:07.642047018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64222112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64224153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64225456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642400422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642419422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"e2397831-0436-430d-924d-0020efe3ddcc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642452633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642462573Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642472553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642482593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642492403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642502693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642539644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51482)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642550734Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35766)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642561284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"ds_connection connected from Ok(127.0.0.1:32986)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.642664205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643238202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643256362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643425294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643607416Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643625426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 77, 69, 76, 74, 74, 60, 71, 70, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643728747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[0]R generation[0..12]: [49, 49, 48, 44, 48, 47, 47, 38, 46, 45, 49, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643740177Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643751537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 77, 69, 76, 74, 74, 60, 71, 70, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643762458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[1]R generation[0..12]: [49, 49, 48, 44, 48, 47, 47, 38, 46, 45, 49, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643773268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643784528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 77, 69, 76, 74, 74, 60, 71, 70, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643795348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[2]R generation[0..12]: [49, 49, 48, 44, 48, 47, 47, 38, 46, 45, 49, 44]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643805838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643817168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643827058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643837298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643847018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643857979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"":"downstairs","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643868029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643877199Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"0","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643887389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"1","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643897379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"client":"2","":"downstairs","session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643907719Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643917339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"e2397831-0436-430d-924d-0020efe3ddcc is now active with session: 47a9559a-13e7-449d-aa71-83e3f8e1932b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.643927739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64393703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"session_id":"47a9559a-13e7-449d-aa71-83e3f8e1932b"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64394668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Activated sub_volume e2397831-0436-430d-924d-0020efe3ddcc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64395515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.64396369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.644397835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} 01/30 Write block 459 len 12288 data: 5 5 5 02/30 Write block 571 len 32768 data: 4 6 8 9 11 12 10 13 03/30 Write block 79 len 8192 data: 4 6 04/30 Write block 590 len 12288 data: 3 3 3 05/30 Read block 381 len 36864 06/30 Read block 443 len 36864 07/30 Write block 338 len 24576 data: 4 5 5 6 6 5 08/30 Write block 126 len 40960 data: 5 7 8 7 7 6 7 8 8 5 09/30 Read block 526 len 32768 10/30 Flush 11/30 Write block 216 len 36864 data: 4 5 3 2 2 2 2 3 3 12/30 Flush 13/30 Write block 214 len 28672 data: 3 3 5 6 4 3 3 14/30 Read block 346 len 20480 15/30 Read block 582 len 12288 16/30 Write block 499 len 24576 data: 5 6 6 5 4 4 17/30 Flush 18/30 Write block 288 len 36864 data: 4 4 4 3 4 4 3 4 6 19/30 Write block 138 len 32768 data: 5 4 3 3 3 3 3 3 20/30 Write block 557 len 16384 data: 6 8 9 7 21/30 Write block 273 len 36864 data: 7 7 9 9 10 11 7 5 3 22/30 Read block 148 len 28672 23/30 Read block 272 len 4096 24/30 Read block 590 len 40960 25/30 Read block 302 len 24576 26/30 Read block 287 len 32768 27/30 Read block 359 len 12288 28/30 Write block 128 len 20480 data: 9 8 8 7 8 29/30 Read block 511 len 16384 30/30 Write block 125 len 8192 data: 4 6 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:07.653367597Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:07.6536151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1333,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 44 50 79 69 79 F F F <--- 4 080-099 50 48 50 79 76 79 F F F <--- 5 100-119 47 47 47 74 74 74 F F F 6 120-139 50 47 50 79 74 79 T F T <--- 7 140-159 38 38 38 60 60 60 T F T <--- 8 160-179 46 46 46 71 71 71 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 50 49 50 81 78 81 F F F <--- 11 220-239 50 44 50 81 69 81 F F F <--- 12 240-259 46 46 46 71 71 71 F F F 13 260-279 48 48 48 75 75 75 T F T <--- 14 280-299 47 47 47 74 74 74 T F T <--- 15 300-319 45 45 45 70 70 70 F F F 16 320-339 50 44 50 79 69 79 F F F <--- 17 340-359 50 44 50 79 69 79 F F F <--- 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 50 49 50 79 78 79 F F F <--- 23 460-479 50 46 50 79 71 79 F F F <--- 24 480-499 50 47 50 81 74 81 F F F <--- 25 500-519 50 47 50 81 74 81 F F F <--- 26 520-539 46 46 46 72 72 72 F F F 27 540-559 46 46 46 72 72 72 T F T <--- 28 560-579 50 42 50 79 66 79 T F T <--- 29 580-599 50 48 50 79 76 79 F F F <--- Max gen: 50, Max flush: 81 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 50 50 79 79 79 F F F 4 080-099 50 50 50 79 79 79 F F F 5 100-119 47 47 47 74 74 74 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 46 46 46 71 71 71 F F F 9 180-199 45 45 45 70 70 70 F F F 10 200-219 50 50 50 81 81 81 F F F 11 220-239 50 50 50 81 81 81 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 51 51 51 82 82 82 F F F 14 280-299 51 51 51 82 82 82 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 50 50 50 79 79 79 F F F 17 340-359 50 50 50 79 79 79 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 50 50 50 79 79 79 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 50 50 50 81 81 81 F F F 25 500-519 50 50 50 81 81 81 F F F 26 520-539 46 46 46 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 51, Max flush: 82 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.909412861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:11.916843507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.916916377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.92413449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924382353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924409463Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924418023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924429843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Crucible 99a62f96-33e6-4498-93e9-bca686ff1570 has session id: e74715d1-3f55-46d7-b05c-def9382e1cef","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924723617Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Upstairs opts: Upstairs UUID: 99a62f96-33e6-4498-93e9-bca686ff1570, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924752457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Crucible stats registered with UUID: 99a62f96-33e6-4498-93e9-bca686ff1570","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.924763987Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:11.924773597Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.92497705Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.92501727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.92502863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925120421Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925152922Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"99a62f96-33e6-4498-93e9-bca686ff1570 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925165732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925175162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925187352Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925197442Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925207292Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925217732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"ds_connection connected from Ok(127.0.0.1:38923)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925257133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"ds_connection connected from Ok(127.0.0.1:58185)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925268413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925281953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"ds_connection connected from Ok(127.0.0.1:42414)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925361464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.925992891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926021792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926067112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926304775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926342145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 74, 82, 82, 71, 70, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926386866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[0]R generation[0..12]: [49, 49, 48, 50, 50, 47, 51, 51, 46, 45, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926398446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926411996Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 74, 82, 82, 71, 70, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926422936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[1]R generation[0..12]: [49, 49, 48, 50, 50, 47, 51, 51, 46, 45, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926433556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926444766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 74, 82, 82, 71, 70, 81, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926455527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[2]R generation[0..12]: [49, 49, 48, 50, 50, 47, 51, 51, 46, 45, 50, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926466217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926477447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926487187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926497237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926506827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926516477Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"":"downstairs","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926526357Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926535507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"0","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926545438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"1","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926555478Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"client":"2","":"downstairs","session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926566568Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926576068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"99a62f96-33e6-4498-93e9-bca686ff1570 is now active with session: e74715d1-3f55-46d7-b05c-def9382e1cef","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926586408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926595398Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"session_id":"e74715d1-3f55-46d7-b05c-def9382e1cef"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926604988Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Activated sub_volume 99a62f96-33e6-4498-93e9-bca686ff1570","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926613418Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.926621978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.927068924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} 01/30 Write block 111 len 20480 data: 7 5 6 4 3 02/30 Write block 448 len 8192 data: 3 3 03/30 Read block 451 len 4096 04/30 Read block 197 len 8192 05/30 Read block 588 len 12288 06/30 Read block 28 len 40960 07/30 Read block 248 len 8192 08/30 Read block 370 len 20480 09/30 Read block 155 len 40960 10/30 Write block 208 len 8192 data: 5 5 11/30 Write block 337 len 20480 data: 4 5 6 6 7 12/30 Write block 491 len 16384 data: 5 4 3 3 13/30 Read block 497 len 40960 14/30 Read block 420 len 8192 15/30 Read block 217 len 12288 16/30 Read block 544 len 4096 17/30 Write block 360 len 12288 data: 4 3 2 18/30 Read block 357 len 4096 19/30 Read block 97 len 40960 20/30 Write block 382 len 28672 data: 6 7 6 7 7 8 7 21/30 Flush 22/30 Write block 193 len 24576 data: 8 8 7 7 5 4 23/30 Read block 39 len 36864 24/30 Write block 159 len 36864 data: 3 4 5 5 7 7 7 7 7 25/30 Read block 269 len 40960 26/30 Write block 332 len 24576 data: 6 7 6 4 4 5 27/30 Read block 67 len 24576 28/30 Read block 478 len 16384 29/30 Write block 511 len 40960 data: 5 5 5 3 3 5 5 5 4 4 30/30 Read block 194 len 40960 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 1 0 25 Done 6 29 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:11.935580091Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:11.935858114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1341,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 50 50 79 79 79 F F F 4 080-099 50 50 50 79 79 79 F F F 5 100-119 47 52 52 74 83 83 T F F <--- 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F T T <--- 8 160-179 46 46 46 71 71 71 F T T <--- 9 180-199 45 45 45 70 70 70 F T T <--- 10 200-219 50 52 52 81 83 83 F F F <--- 11 220-239 50 50 50 81 81 81 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 51 51 51 82 82 82 F F F 14 280-299 51 51 51 82 82 82 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 50 52 52 79 83 83 F T T <--- 17 340-359 50 52 52 79 83 83 F F F <--- 18 360-379 46 52 52 73 83 83 F F F <--- 19 380-399 46 52 52 73 83 83 F F F <--- 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 50 52 52 79 83 83 T F F <--- 23 460-479 50 50 50 79 79 79 F F F 24 480-499 50 52 52 81 83 83 F F F <--- 25 500-519 50 50 50 81 81 81 F T T <--- 26 520-539 46 46 46 72 72 72 F T T <--- 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 52, Max flush: 83 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 50 50 79 79 79 F F F 4 080-099 50 50 50 79 79 79 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 53 53 53 84 84 84 F F F 8 160-179 53 53 53 84 84 84 F F F 9 180-199 53 53 53 84 84 84 F F F 10 200-219 52 52 52 83 83 83 F F F 11 220-239 50 50 50 81 81 81 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 51 51 51 82 82 82 F F F 14 280-299 51 51 51 82 82 82 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 53 53 53 84 84 84 F F F 17 340-359 52 52 52 83 83 83 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 52 52 52 83 83 83 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 53 53 53 84 84 84 F F F 26 520-539 53 53 53 84 84 84 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 53, Max flush: 84 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.193588628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:16.201327107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.201410348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.208847743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209137266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209161557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209170787Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209183377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Crucible 7c66987c-5690-405e-892e-d06d4d297343 has session id: 2ba2457a-21e1-477c-abef-baf8de40d22d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209550591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Upstairs opts: Upstairs UUID: 7c66987c-5690-405e-892e-d06d4d297343, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209591712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Crucible stats registered with UUID: 7c66987c-5690-405e-892e-d06d4d297343","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209603062Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:16.209612752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209803114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209837814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209849505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209969326Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.209996486Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"7c66987c-5690-405e-892e-d06d4d297343 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210006256Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210016236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210027147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210037577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210048027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210058937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210069457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63288)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210167558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45338)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210179578Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62744)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210255369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210905147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.210924037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211029968Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211260001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211284381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 84, 84, 83, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211403772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[0]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 53, 53, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211416302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211428153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 84, 84, 83, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211439423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[1]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 53, 53, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211450543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211462123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 84, 84, 83, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211475363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[2]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 53, 53, 52, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211486583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211498323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211508404Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211518944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211528984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211539114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"":"downstairs","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211549494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211559014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"0","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211569704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"1","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211579984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"client":"2","":"downstairs","session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211590704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211600605Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"7c66987c-5690-405e-892e-d06d4d297343 is now active with session: 2ba2457a-21e1-477c-abef-baf8de40d22d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211611345Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211620865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"session_id":"2ba2457a-21e1-477c-abef-baf8de40d22d"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211630715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Activated sub_volume 7c66987c-5690-405e-892e-d06d4d297343","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211640335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.211649085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.21209522Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} 01/30 Write block 448 len 24576 data: 4 4 6 4 3 4 02/30 Write block 196 len 40960 data: 8 6 5 3 3 5 7 6 7 6 03/30 Write block 164 len 40960 data: 8 8 8 8 7 6 5 3 3 5 04/30 Write block 210 len 32768 data: 5 7 5 3 4 4 6 7 05/30 Read block 410 len 16384 06/30 Write block 289 len 32768 data: 5 5 4 5 5 4 5 7 07/30 Read block 59 len 20480 08/30 Read block 144 len 32768 09/30 Read block 139 len 4096 10/30 Read block 336 len 28672 11/30 Read block 70 len 20480 12/30 Write block 180 len 16384 data: 7 6 5 6 13/30 Read block 376 len 16384 14/30 Write block 574 len 12288 data: 10 12 13 15/30 Flush 16/30 Write block 369 len 12288 data: 4 3 3 17/30 Read block 290 len 32768 18/30 Read block 254 len 16384 19/30 Write block 359 len 20480 data: 4 5 4 3 3 20/30 Write block 504 len 16384 data: 5 4 4 4 21/30 Read block 461 len 40960 22/30 Read block 585 len 40960 23/30 Read block 332 len 16384 24/30 Read block 535 len 8192 25/30 Read block 535 len 16384 26/30 Write block 192 len 32768 data: 8 9 9 8 9 7 6 4 27/30 Write block 323 len 40960 data: 6 4 4 5 5 5 4 3 3 7 28/30 Read block 311 len 36864 29/30 Write block 343 len 40960 data: 6 5 5 4 4 3 4 3 4 3 30/30 Read block 508 len 32768 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:16.221595549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:16.221877702Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1349,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 50 50 79 79 79 F F F 4 080-099 50 50 50 79 79 79 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 53 53 53 84 84 84 F F F 8 160-179 53 54 54 84 85 85 F F F <--- 9 180-199 53 54 54 84 85 85 T T T <--- 10 200-219 52 54 54 83 85 85 T F F <--- 11 220-239 50 50 50 81 81 81 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 51 51 51 82 82 82 F F F 14 280-299 51 54 54 82 85 85 F F F <--- 15 300-319 45 45 45 70 70 70 F F F 16 320-339 53 53 53 84 84 84 F T T <--- 17 340-359 52 52 52 83 83 83 F T T <--- 18 360-379 52 52 52 83 83 83 F T T <--- 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 52 54 54 83 85 85 T F F <--- 23 460-479 50 50 50 79 79 79 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 53 53 53 84 84 84 F T T <--- 26 520-539 53 53 53 84 84 84 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 54 54 82 85 85 F F F <--- 29 580-599 50 50 50 79 79 79 F F F Max gen: 54, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 49 49 49 78 78 78 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 50 50 50 79 79 79 F F F 4 080-099 50 50 50 79 79 79 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 53 53 53 84 84 84 F F F 8 160-179 54 54 54 85 85 85 F F F 9 180-199 55 55 55 86 86 86 F F F 10 200-219 54 54 54 85 85 85 F F F 11 220-239 50 50 50 81 81 81 F F F 12 240-259 46 46 46 71 71 71 F F F 13 260-279 51 51 51 82 82 82 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 45 45 45 70 70 70 F F F 16 320-339 55 55 55 86 86 86 F F F 17 340-359 55 55 55 86 86 86 F F F 18 360-379 55 55 55 86 86 86 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 44 44 44 69 69 69 F F F 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 55 55 55 86 86 86 F F F 26 520-539 53 53 53 84 84 84 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 55, Max flush: 86 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.47204857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.480696169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.480991342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481015513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481024893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481036783Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Crucible 5aa23a9a-7eac-481f-a122-d193b96d9be4 has session id: c12f14d6-d5de-45fb-ab8c-ca01af7c86a1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481426707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Upstairs opts: Upstairs UUID: 5aa23a9a-7eac-481f-a122-d193b96d9be4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481456558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Crucible stats registered with UUID: 5aa23a9a-7eac-481f-a122-d193b96d9be4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481468248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:20.481479518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.48167339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.48170487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481716221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481813962Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481838552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"5aa23a9a-7eac-481f-a122-d193b96d9be4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481922273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481932233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481942523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481952503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481962773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481973114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.481983364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61403)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482058904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59124)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482099485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:60428)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482116585Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482654061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482684062Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482794883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482948275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.482965425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 85, 86, 85, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483065356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[0]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 54, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483076856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483088386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 85, 86, 85, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483099236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[1]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 54, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483109977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483121447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 77, 79, 79, 83, 82, 84, 85, 86, 85, 81]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483132117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[2]R generation[0..12]: [49, 49, 48, 50, 50, 52, 51, 53, 54, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483142877Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483154127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483163917Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483173867Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483183537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483194868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"":"downstairs","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483205108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483214288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"0","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483224228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"1","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483234238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"client":"2","":"downstairs","session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483244438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483254078Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"5aa23a9a-7eac-481f-a122-d193b96d9be4 is now active with session: c12f14d6-d5de-45fb-ab8c-ca01af7c86a1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483264378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483273488Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"session_id":"c12f14d6-d5de-45fb-ab8c-ca01af7c86a1"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483282909Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Activated sub_volume 5aa23a9a-7eac-481f-a122-d193b96d9be4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483291479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483300029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.483680313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} 01/30 Write block 65 len 20480 data: 6 6 6 5 4 02/30 Read block 260 len 12288 03/30 Read block 404 len 8192 04/30 Read block 9 len 28672 05/30 Read block 134 len 24576 06/30 Write block 153 len 20480 data: 2 3 2 2 2 07/30 Read block 581 len 24576 08/30 Read block 373 len 8192 09/30 Write block 92 len 8192 data: 6 6 10/30 Read block 587 len 8192 11/30 Read block 417 len 40960 12/30 Write block 433 len 16384 data: 8 8 8 9 13/30 Read block 361 len 32768 14/30 Flush 15/30 Write block 251 len 12288 data: 7 6 7 16/30 Write block 32 len 24576 data: 3 4 3 3 3 3 17/30 Read block 232 len 28672 18/30 Write block 262 len 8192 data: 11 12 19/30 Read block 409 len 40960 20/30 Write block 215 len 32768 data: 5 7 8 5 4 4 3 3 21/30 Write block 260 len 36864 data: 10 10 12 13 10 11 8 7 6 22/30 Write block 303 len 28672 data: 10 10 8 6 6 5 5 23/30 Read block 279 len 32768 24/30 Read block 502 len 36864 25/30 Flush 26/30 Write block 512 len 8192 data: 6 6 27/30 Read block 553 len 40960 28/30 Read block 542 len 16384 29/30 Write block 174 len 36864 data: 5 6 6 5 5 5 8 7 6 30/30 Write block 513 len 32768 data: 7 4 4 6 6 6 5 5 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 2 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:20.492349842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:20.492624785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 56 56 49 88 88 78 F F F <--- 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 50 87 87 79 F F T <--- 4 080-099 56 56 50 87 87 79 F F F <--- 5 100-119 52 52 52 83 83 83 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 56 56 53 87 87 84 F F F <--- 8 160-179 54 54 54 85 85 85 T T F <--- 9 180-199 55 55 55 86 86 86 T T F <--- 10 200-219 56 56 54 88 88 85 F F F <--- 11 220-239 56 56 50 88 88 81 F F F <--- 12 240-259 56 56 46 88 88 71 F F F <--- 13 260-279 56 56 51 88 88 82 F F F <--- 14 280-299 54 54 54 85 85 85 F F F 15 300-319 56 56 45 88 88 70 F F F <--- 16 320-339 55 55 55 86 86 86 F F F 17 340-359 55 55 55 86 86 86 F F F 18 360-379 55 55 55 86 86 86 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 56 56 44 87 87 69 F F F <--- 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 55 55 55 86 86 86 T T F <--- 26 520-539 53 53 53 84 84 84 T T F <--- 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 56, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 78 78 78 F F F 1 020-039 56 56 56 88 88 88 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 56 56 56 87 87 87 F F F 8 160-179 57 57 57 89 89 89 F F F 9 180-199 57 57 57 89 89 89 F F F 10 200-219 56 56 56 88 88 88 F F F 11 220-239 56 56 56 88 88 88 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 56 56 56 88 88 88 F F F 16 320-339 55 55 55 86 86 86 F F F 17 340-359 55 55 55 86 86 86 F F F 18 360-379 55 55 55 86 86 86 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 56 56 56 87 87 87 F F F 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 57 57 57 89 89 89 F F F 26 520-539 57 57 57 89 89 89 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 50 50 50 79 79 79 F F F Max gen: 57, Max flush: 89 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.737877617Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.746386884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.746685348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.746708348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.746720028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.746731858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Crucible 9424506c-845d-4465-ac46-276542dee35d has session id: 1ff4eb44-8f99-4612-9bad-c410622e944f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747082992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Upstairs opts: Upstairs UUID: 9424506c-845d-4465-ac46-276542dee35d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747110423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Crucible stats registered with UUID: 9424506c-845d-4465-ac46-276542dee35d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747120883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:24.747134683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747318845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747349105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747360435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747519127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747543627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"9424506c-845d-4465-ac46-276542dee35d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747556148Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747565498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747575508Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747587358Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747597648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747625538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747637649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54680)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.74776262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50607)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747863081Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50149)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.747875291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748426948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748454548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748589649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748734381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748751841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[0]R flush_numbers[0..12]: [78, 88, 77, 87, 87, 83, 82, 87, 89, 89, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748872403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[0]R generation[0..12]: [49, 56, 48, 56, 56, 52, 51, 56, 57, 57, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748884033Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748895553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[1]R flush_numbers[0..12]: [78, 88, 77, 87, 87, 83, 82, 87, 89, 89, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748908413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[1]R generation[0..12]: [49, 56, 48, 56, 56, 52, 51, 56, 57, 57, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748919163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748930313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[2]R flush_numbers[0..12]: [78, 88, 77, 87, 87, 83, 82, 87, 89, 89, 88, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748941063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[2]R generation[0..12]: [49, 56, 48, 56, 56, 52, 51, 56, 57, 57, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748951864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748963114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748972944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748982944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.748992594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749002404Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"":"downstairs","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749012474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749021724Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"0","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749031815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"1","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749041925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"client":"2","":"downstairs","session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749052165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749062865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"9424506c-845d-4465-ac46-276542dee35d is now active with session: 1ff4eb44-8f99-4612-9bad-c410622e944f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749073045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749082165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"session_id":"1ff4eb44-8f99-4612-9bad-c410622e944f"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749091575Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Activated sub_volume 9424506c-845d-4465-ac46-276542dee35d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749100005Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.749108515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.74951541Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} 01/30 Write block 597 len 4096 data: 3 02/30 Read block 262 len 8192 03/30 Write block 10 len 28672 data: 5 6 5 5 5 4 4 04/30 Write block 160 len 4096 data: 5 05/30 Write block 199 len 12288 data: 5 4 6 06/30 Read block 85 len 4096 07/30 Read block 353 len 36864 08/30 Read block 519 len 20480 09/30 Read block 422 len 8192 10/30 Read block 279 len 32768 11/30 Read block 453 len 20480 12/30 Flush 13/30 Write block 171 len 36864 data: 4 4 6 6 7 7 6 6 6 14/30 Read block 248 len 36864 15/30 Read block 214 len 28672 16/30 Write block 484 len 20480 data: 5 4 3 3 2 17/30 Read block 414 len 16384 18/30 Read block 282 len 4096 19/30 Read block 247 len 4096 20/30 Read block 411 len 4096 21/30 Read block 78 len 32768 22/30 Read block 99 len 20480 23/30 Write block 185 len 40960 data: 8 8 6 5 4 5 7 9 10 10 24/30 Flush 25/30 Write block 120 len 4096 data: 3 26/30 Flush 27/30 Write block 374 len 4096 data: 3 28/30 Read block 396 len 36864 29/30 Write block 514 len 36864 data: 5 5 7 7 7 6 6 4 3 30/30 Read block 387 len 12288 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 24 24 Done 30 30 6 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:24.757618653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:24.757891266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1365,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 49 90 90 78 F F T <--- 1 020-039 56 56 56 88 88 88 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 58 58 51 92 92 82 F F F <--- 7 140-159 56 56 56 87 87 87 F F F 8 160-179 58 58 57 91 91 89 F F T <--- 9 180-199 58 58 57 91 91 89 F F T <--- 10 200-219 58 58 56 90 90 88 F F T <--- 11 220-239 56 56 56 88 88 88 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 56 56 56 88 88 88 F F F 16 320-339 55 55 55 86 86 86 F F F 17 340-359 55 55 55 86 86 86 F F F 18 360-379 55 55 55 86 86 86 T T F <--- 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 56 56 56 87 87 87 F F F 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 58 58 52 91 91 83 F F F <--- 25 500-519 57 57 57 89 89 89 T T F <--- 26 520-539 57 57 57 89 89 89 T T F <--- 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 58 58 50 90 90 79 F F T <--- Max gen: 58, Max flush: 92 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 90 90 90 F F F 1 020-039 56 56 56 88 88 88 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 58 58 58 92 92 92 F F F 7 140-159 56 56 56 87 87 87 F F F 8 160-179 58 58 58 91 91 91 F F F 9 180-199 58 58 58 91 91 91 F F F 10 200-219 58 58 58 90 90 90 F F F 11 220-239 56 56 56 88 88 88 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 56 56 56 88 88 88 F F F 16 320-339 55 55 55 86 86 86 F F F 17 340-359 55 55 55 86 86 86 F F F 18 360-379 59 59 59 93 93 93 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F F F 21 420-439 56 56 56 87 87 87 F F F 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 58 58 58 91 91 91 F F F 25 500-519 59 59 59 93 93 93 F F F 26 520-539 59 59 59 93 93 93 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 58 58 58 90 90 90 F F F Max gen: 59, Max flush: 93 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:28.996252978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:29.003870095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.003948176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011190558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011451101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011474882Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011484262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011496302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Crucible 743e7f68-34e2-401c-93eb-810327d36dbf has session id: 15497b03-3fb3-46bc-af04-97a4671e999f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011868806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Upstairs opts: Upstairs UUID: 743e7f68-34e2-401c-93eb-810327d36dbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011899307Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Crucible stats registered with UUID: 743e7f68-34e2-401c-93eb-810327d36dbf","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.011913157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:29.011923217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012105649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012135709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012146999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012302411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012319771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"743e7f68-34e2-401c-93eb-810327d36dbf active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012343792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012353872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012364282Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012374722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012385022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012395592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012406122Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63318)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012527244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61253)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012538854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54539)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.012608145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013233502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013245772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013349883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013569386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013593126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[0]R flush_numbers[0..12]: [90, 88, 77, 87, 87, 83, 92, 87, 91, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013681177Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[0]R generation[0..12]: [58, 56, 48, 56, 56, 52, 58, 56, 58, 58, 58, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013693407Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013705287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[1]R flush_numbers[0..12]: [90, 88, 77, 87, 87, 83, 92, 87, 91, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013716377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[1]R generation[0..12]: [58, 56, 48, 56, 56, 52, 58, 56, 58, 58, 58, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013727258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013738928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[2]R flush_numbers[0..12]: [90, 88, 77, 87, 87, 83, 92, 87, 91, 91, 90, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013750038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[2]R generation[0..12]: [58, 56, 48, 56, 56, 52, 58, 56, 58, 58, 58, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013760948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013774898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013784938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013795088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013805118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013814979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"":"downstairs","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013825679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013835259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"0","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013846049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"1","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013856549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"client":"2","":"downstairs","session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013867339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013877239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"743e7f68-34e2-401c-93eb-810327d36dbf is now active with session: 15497b03-3fb3-46bc-af04-97a4671e999f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013887789Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.013897049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"session_id":"15497b03-3fb3-46bc-af04-97a4671e999f"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.01390664Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Activated sub_volume 743e7f68-34e2-401c-93eb-810327d36dbf","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.01391602Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.01392464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.014358895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} 01/30 Write block 496 len 28672 data: 4 4 5 6 7 7 6 02/30 Write block 353 len 24576 data: 3 4 3 3 3 4 03/30 Read block 40 len 20480 04/30 Read block 273 len 32768 05/30 Flush 06/30 Write block 129 len 24576 data: 9 9 8 9 9 9 07/30 Read block 448 len 12288 08/30 Write block 311 len 32768 data: 5 5 5 4 4 6 6 5 09/30 Write block 185 len 32768 data: 9 9 7 6 5 6 8 10 10/30 Read block 588 len 8192 11/30 Flush 12/30 Write block 149 len 32768 data: 2 2 2 2 3 4 3 3 13/30 Write block 222 len 40960 data: 4 4 4 4 4 5 5 5 4 4 14/30 Write block 129 len 32768 data: 10 10 9 10 10 10 6 4 15/30 Write block 1 len 12288 data: 3 3 4 16/30 Write block 435 len 4096 data: 9 17/30 Read block 117 len 36864 18/30 Read block 260 len 20480 19/30 Flush 20/30 Write block 492 len 16384 data: 5 4 4 3 21/30 Write block 339 len 28672 data: 7 7 8 7 7 6 6 22/30 Read block 260 len 36864 23/30 Write block 412 len 20480 data: 6 7 7 6 6 24/30 Write block 593 len 12288 data: 3 2 2 25/30 Read block 376 len 12288 26/30 Write block 497 len 40960 data: 5 6 7 8 8 7 5 6 5 5 27/30 Write block 206 len 36864 data: 4 3 6 6 6 8 6 4 5 28/30 Write block 105 len 32768 data: 10 10 8 7 8 7 8 6 29/30 Write block 341 len 16384 data: 9 8 8 7 30/30 Write block 99 len 20480 data: 5 6 7 8 8 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 10 Sent Sent Done false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 5 4 38 Done 1 25 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:29.022373797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:29.022633019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 60 60 90 96 96 F F F <--- 1 020-039 56 56 56 88 88 88 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 56 56 56 87 87 87 F T T <--- 5 100-119 52 52 52 83 83 83 F T T <--- 6 120-139 58 60 60 92 96 96 F F F <--- 7 140-159 56 60 60 87 96 96 F F F <--- 8 160-179 58 58 58 91 91 91 F F F 9 180-199 58 60 60 91 95 95 F F F <--- 10 200-219 58 58 58 90 90 90 F T T <--- 11 220-239 56 60 60 88 96 96 F F F <--- 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 56 60 60 88 95 95 F F F <--- 16 320-339 55 55 55 86 86 86 F T T <--- 17 340-359 55 60 60 86 94 94 F T T <--- 18 360-379 59 59 59 93 93 93 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 46 46 46 72 72 72 F T T <--- 21 420-439 56 60 60 87 96 96 F F F <--- 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 58 60 60 91 94 94 T T T <--- 25 500-519 59 60 60 93 94 94 T T T <--- 26 520-539 59 59 59 93 93 93 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 58 58 58 90 90 90 F T T <--- Max gen: 60, Max flush: 96 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 96 96 96 F F F 1 020-039 56 56 56 88 88 88 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 61 61 61 97 97 97 F F F 5 100-119 61 61 61 97 97 97 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 60 60 60 96 96 96 F F F 8 160-179 58 58 58 91 91 91 F F F 9 180-199 60 60 60 95 95 95 F F F 10 200-219 61 61 61 97 97 97 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 60 60 60 95 95 95 F F F 16 320-339 61 61 61 97 97 97 F F F 17 340-359 61 61 61 97 97 97 F F F 18 360-379 59 59 59 93 93 93 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 60 60 60 96 96 96 F F F 22 440-459 54 54 54 85 85 85 F F F 23 460-479 50 50 50 79 79 79 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 61 61 61 97 97 97 F F F 26 520-539 59 59 59 93 93 93 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 61, Max flush: 97 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.288610958Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.29749095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.297805743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.297829504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.297838554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.297850534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Crucible 9214143b-7f54-441d-bc42-9b405c6d349c has session id: f6927bce-cf07-4799-883e-46357f3fe825","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298224618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Upstairs opts: Upstairs UUID: 9214143b-7f54-441d-bc42-9b405c6d349c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298282449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Crucible stats registered with UUID: 9214143b-7f54-441d-bc42-9b405c6d349c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298294479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:33.298306829Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298500341Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298515931Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298526632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298671083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298695904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"9214143b-7f54-441d-bc42-9b405c6d349c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298709284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298718374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298728374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298738294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298748144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298758544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298796685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"ds_connection connected from Ok(127.0.0.1:36784)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298856705Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"ds_connection connected from Ok(127.0.0.1:46630)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298868676Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"ds_connection connected from Ok(127.0.0.1:65476)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.298879516Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.299608754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.299632544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.299692535Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.299917778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.299955308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[0]R flush_numbers[0..12]: [96, 88, 77, 87, 97, 97, 96, 96, 91, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300006629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[0]R generation[0..12]: [60, 56, 48, 56, 61, 61, 60, 60, 58, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300018119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300029669Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[1]R flush_numbers[0..12]: [96, 88, 77, 87, 97, 97, 96, 96, 91, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300040459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[1]R generation[0..12]: [60, 56, 48, 56, 61, 61, 60, 60, 58, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300051259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300062459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[2]R flush_numbers[0..12]: [96, 88, 77, 87, 97, 97, 96, 96, 91, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300073299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[2]R generation[0..12]: [60, 56, 48, 56, 61, 61, 60, 60, 58, 60, 61, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300083919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30009524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30010529Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30011528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30012495Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30013471Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"":"downstairs","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30014639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30015547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"0","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30016534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"1","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.30017523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"client":"2","":"downstairs","session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300185451Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300194941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"9214143b-7f54-441d-bc42-9b405c6d349c is now active with session: f6927bce-cf07-4799-883e-46357f3fe825","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300205161Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300214381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"session_id":"f6927bce-cf07-4799-883e-46357f3fe825"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300223891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Activated sub_volume 9214143b-7f54-441d-bc42-9b405c6d349c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300232531Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300241201Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.300630946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} 01/30 Write block 511 len 4096 data: 6 02/30 Read block 330 len 24576 03/30 Read block 359 len 16384 04/30 Write block 459 len 36864 data: 6 6 6 5 4 4 2 3 3 05/30 Read block 159 len 12288 06/30 Read block 186 len 24576 07/30 Write block 302 len 24576 data: 8 11 11 9 7 7 08/30 Write block 15 len 4096 data: 5 09/30 Read block 154 len 40960 10/30 Write block 204 len 8192 data: 8 7 11/30 Read block 518 len 20480 12/30 Write block 45 len 36864 data: 4 4 4 5 4 4 6 5 6 13/30 Read block 204 len 8192 14/30 Write block 538 len 4096 data: 8 15/30 Flush 16/30 Write block 513 len 4096 data: 8 17/30 Read block 47 len 4096 18/30 Read block 583 len 28672 19/30 Write block 85 len 20480 data: 4 4 4 4 3 20/30 Write block 507 len 32768 data: 5 4 5 6 7 7 9 6 21/30 Read block 354 len 8192 22/30 Flush 23/30 Write block 553 len 8192 data: 4 4 24/30 Read block 541 len 28672 25/30 Write block 508 len 40960 data: 5 6 7 8 8 10 7 6 8 8 26/30 Read block 40 len 20480 27/30 Read block 353 len 28672 28/30 Write block 146 len 16384 data: 4 3 2 3 29/30 Write block 536 len 8192 data: 6 6 30/30 Write block 100 len 28672 data: 7 8 9 9 8 11 11 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 29 35 Done 27 27 1 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:33.30890941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:33.309183534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1381,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 60 98 98 96 F F F <--- 1 020-039 56 56 56 88 88 88 F F F 2 040-059 62 62 48 98 98 77 F F F <--- 3 060-079 56 56 56 87 87 87 F F F 4 080-099 62 62 61 99 99 97 F F F <--- 5 100-119 61 61 61 97 97 97 T T F <--- 6 120-139 60 60 60 96 96 96 F F F 7 140-159 60 60 60 96 96 96 T T F <--- 8 160-179 58 58 58 91 91 91 F F F 9 180-199 60 60 60 95 95 95 F F F 10 200-219 62 62 61 98 98 97 F F F <--- 11 220-239 60 60 60 96 96 96 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 62 62 60 98 98 95 F F F <--- 16 320-339 61 61 61 97 97 97 F F F 17 340-359 61 61 61 97 97 97 F F F 18 360-379 59 59 59 93 93 93 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 60 60 60 96 96 96 F F F 22 440-459 62 62 54 98 98 85 F F F <--- 23 460-479 62 62 50 98 98 79 F F F <--- 24 480-499 61 61 61 97 97 97 F F F 25 500-519 62 62 61 99 99 97 T T T <--- 26 520-539 62 62 59 98 98 93 T T F <--- 27 540-559 51 51 51 82 82 82 T T F <--- 28 560-579 54 54 54 85 85 85 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 62, Max flush: 99 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 56 56 56 88 88 88 F F F 2 040-059 62 62 62 98 98 98 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 62 62 62 99 99 99 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 58 58 58 91 91 91 F F F 9 180-199 60 60 60 95 95 95 F F F 10 200-219 62 62 62 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 F F F 14 280-299 54 54 54 85 85 85 F F F 15 300-319 62 62 62 98 98 98 F F F 16 320-339 61 61 61 97 97 97 F F F 17 340-359 61 61 61 97 97 97 F F F 18 360-379 59 59 59 93 93 93 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 60 60 60 96 96 96 F F F 22 440-459 62 62 62 98 98 98 F F F 23 460-479 62 62 62 98 98 98 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 63 63 63 100 100 100 F F F 26 520-539 63 63 63 100 100 100 F F F 27 540-559 63 63 63 100 100 100 F F F 28 560-579 54 54 54 85 85 85 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 63, Max flush: 100 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.553383963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.561700838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.561997322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562020912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562036772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562048892Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Crucible 558967d7-d555-4b05-9d29-5ceb9108eeea has session id: a15feffd-9032-401e-8fea-e31cc3e59043","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562408526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Upstairs opts: Upstairs UUID: 558967d7-d555-4b05-9d29-5ceb9108eeea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562438247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Crucible stats registered with UUID: 558967d7-d555-4b05-9d29-5ceb9108eeea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562449507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:37.562459807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562634179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562664529Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562675849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562822121Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562848661Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"558967d7-d555-4b05-9d29-5ceb9108eeea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562859111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562868312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562878432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562888482Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562898502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562910552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.562920972Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"ds_connection connected from Ok(127.0.0.1:61062)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563020423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"ds_connection connected from Ok(127.0.0.1:41320)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563046224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"ds_connection connected from Ok(127.0.0.1:57276)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563057914Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563767542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563812982Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.563832983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564118846Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564141846Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[0]R flush_numbers[0..12]: [98, 88, 98, 87, 99, 100, 96, 100, 91, 95, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564152556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[0]R generation[0..12]: [62, 56, 62, 56, 62, 63, 60, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564163936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564175266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[1]R flush_numbers[0..12]: [98, 88, 98, 87, 99, 100, 96, 100, 91, 95, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564186287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[1]R generation[0..12]: [62, 56, 62, 56, 62, 63, 60, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564199347Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564210857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[2]R flush_numbers[0..12]: [98, 88, 98, 87, 99, 100, 96, 100, 91, 95, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564221707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[2]R generation[0..12]: [62, 56, 62, 56, 62, 63, 60, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564232537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564243757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564253547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564263717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564273438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564283128Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"":"downstairs","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564293188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564302498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"0","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564312638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"1","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564322788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"client":"2","":"downstairs","session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564333108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564342648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"558967d7-d555-4b05-9d29-5ceb9108eeea is now active with session: a15feffd-9032-401e-8fea-e31cc3e59043","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564352949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564363119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"session_id":"a15feffd-9032-401e-8fea-e31cc3e59043"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564372609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Activated sub_volume 558967d7-d555-4b05-9d29-5ceb9108eeea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564380949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564389519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.564804514Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} 01/30 Flush 02/30 Write block 21 len 36864 data: 5 4 3 3 3 3 3 4 4 03/30 Flush 04/30 Write block 426 len 20480 data: 3 4 4 5 9 05/30 Read block 251 len 32768 06/30 Read block 414 len 32768 07/30 Write block 300 len 28672 data: 6 8 9 12 12 10 8 08/30 Write block 299 len 28672 data: 6 7 9 10 13 13 11 09/30 Write block 548 len 4096 data: 5 10/30 Read block 233 len 32768 11/30 Read block 202 len 40960 12/30 Read block 426 len 24576 13/30 Flush 14/30 Write block 556 len 40960 data: 4 7 9 10 8 7 8 8 7 7 15/30 Write block 391 len 12288 data: 6 6 6 16/30 Read block 294 len 20480 17/30 Read block 309 len 16384 18/30 Read block 220 len 12288 19/30 Read block 482 len 8192 20/30 Write block 309 len 20480 data: 6 5 6 6 6 21/30 Read block 309 len 20480 22/30 Read block 54 len 20480 23/30 Write block 369 len 4096 data: 5 24/30 Write block 262 len 8192 data: 13 14 25/30 Read block 519 len 8192 26/30 Read block 74 len 20480 27/30 Read block 358 len 20480 28/30 Read block 190 len 36864 29/30 Write block 271 len 12288 data: 5 5 8 30/30 Write block 178 len 40960 data: 7 7 9 8 7 7 6 10 10 8 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 2 31 Done 28 1 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1000)) Some(JobId(1012)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:37.57318162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:37.573490423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1389,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 56 64 102 88 102 F T F <--- 2 040-059 62 62 62 98 98 98 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 62 62 62 99 99 99 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 58 58 58 91 91 91 T F T <--- 9 180-199 60 60 60 95 95 95 T F T <--- 10 200-219 62 62 62 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 56 56 56 88 88 88 T F T <--- 14 280-299 64 54 64 103 85 103 F F F <--- 15 300-319 64 62 64 103 98 103 T F T <--- 16 320-339 61 61 61 97 97 97 F F F 17 340-359 61 61 61 97 97 97 F F F 18 360-379 59 59 59 93 93 93 T F T <--- 19 380-399 52 52 52 83 83 83 T F T <--- 20 400-419 61 61 61 97 97 97 F F F 21 420-439 64 60 64 103 96 103 F F F <--- 22 440-459 62 62 62 98 98 98 F F F 23 460-479 62 62 62 98 98 98 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 63 63 63 100 100 100 F F F 26 520-539 63 63 63 100 100 100 F F F 27 540-559 64 63 64 103 100 103 T F T <--- 28 560-579 54 54 54 85 85 85 T F T <--- 29 580-599 61 61 61 97 97 97 F F F Max gen: 64, Max flush: 103 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 62 62 62 98 98 98 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 62 62 62 99 99 99 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 65 65 65 104 104 104 F F F 9 180-199 65 65 65 104 104 104 F F F 10 200-219 62 62 62 98 98 98 F F F 11 220-239 60 60 60 96 96 96 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 65 65 65 104 104 104 F F F 14 280-299 64 64 64 103 103 103 F F F 15 300-319 65 65 65 104 104 104 F F F 16 320-339 61 61 61 97 97 97 F F F 17 340-359 61 61 61 97 97 97 F F F 18 360-379 65 65 65 104 104 104 F F F 19 380-399 65 65 65 104 104 104 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 64 64 64 103 103 103 F F F 22 440-459 62 62 62 98 98 98 F F F 23 460-479 62 62 62 98 98 98 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 63 63 63 100 100 100 F F F 26 520-539 63 63 63 100 100 100 F F F 27 540-559 65 65 65 104 104 104 F F F 28 560-579 65 65 65 104 104 104 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 65, Max flush: 104 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.813587055Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:41.821004799Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.82108249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828303603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828557176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828585296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828594236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828605966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Crucible 74ff7c36-ef29-4e95-92ba-f1d5acebe588 has session id: d3ad2a68-8a13-4084-a28b-55b7a94f83eb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.828961231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Upstairs opts: Upstairs UUID: 74ff7c36-ef29-4e95-92ba-f1d5acebe588, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829009911Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Crucible stats registered with UUID: 74ff7c36-ef29-4e95-92ba-f1d5acebe588","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829023801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:41.829033771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829204923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829234774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829245964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829393035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829411856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"74ff7c36-ef29-4e95-92ba-f1d5acebe588 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829429846Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829439126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829449036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829459116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829469206Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829479636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829489937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59794)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829557557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63049)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829655688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62702)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.829684679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830191175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830214695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830408837Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830498288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830514278Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[0]R flush_numbers[0..12]: [98, 102, 98, 87, 99, 100, 96, 100, 104, 104, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.83069992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[0]R generation[0..12]: [62, 64, 62, 56, 62, 63, 60, 63, 65, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830711731Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830723331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[1]R flush_numbers[0..12]: [98, 102, 98, 87, 99, 100, 96, 100, 104, 104, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830734171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[1]R generation[0..12]: [62, 64, 62, 56, 62, 63, 60, 63, 65, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830744961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830756351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[2]R flush_numbers[0..12]: [98, 102, 98, 87, 99, 100, 96, 100, 104, 104, 98, 96]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830767221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[2]R generation[0..12]: [62, 64, 62, 56, 62, 63, 60, 63, 65, 65, 62, 60]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830777881Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830791101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830801032Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830811192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830821052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830830872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"":"downstairs","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830841072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830850192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"0","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830860262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"1","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830870432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"client":"2","":"downstairs","session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830880782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830890443Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"74ff7c36-ef29-4e95-92ba-f1d5acebe588 is now active with session: d3ad2a68-8a13-4084-a28b-55b7a94f83eb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830900913Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830910223Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"session_id":"d3ad2a68-8a13-4084-a28b-55b7a94f83eb"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830919943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Activated sub_volume 74ff7c36-ef29-4e95-92ba-f1d5acebe588","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830928613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.830937253Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.831318268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} 01/30 Flush 02/30 Write block 178 len 12288 data: 8 8 10 03/30 Read block 125 len 36864 04/30 Read block 232 len 12288 05/30 Read block 472 len 40960 06/30 Write block 436 len 36864 data: 10 10 10 10 6 6 7 5 5 07/30 Read block 347 len 8192 08/30 Read block 53 len 8192 09/30 Write block 317 len 36864 data: 7 6 5 5 5 6 7 5 5 10/30 Flush 11/30 Write block 440 len 28672 data: 7 7 8 6 6 4 4 12/30 Write block 354 len 32768 data: 5 4 4 4 5 5 6 5 13/30 Read block 169 len 16384 14/30 Read block 441 len 8192 15/30 Write block 84 len 20480 data: 4 5 5 5 5 16/30 Read block 265 len 40960 17/30 Write block 231 len 32768 data: 5 5 5 4 5 5 4 5 18/30 Write block 265 len 8192 data: 12 9 19/30 Flush 20/30 Write block 169 len 24576 data: 7 6 5 5 7 7 21/30 Read block 160 len 8192 22/30 Read block 525 len 8192 23/30 Read block 448 len 12288 24/30 Read block 92 len 24576 25/30 Write block 278 len 32768 data: 12 8 6 4 4 4 5 5 26/30 Write block 43 len 12288 data: 4 3 5 27/30 Write block 167 len 4096 data: 9 28/30 Write block 299 len 16384 data: 7 8 10 11 29/30 Read block 94 len 4096 30/30 Write block 533 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 1 Sent Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 2 31 Done 1 28 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:41.839784124Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:41.840072048Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1397,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 62 62 62 98 98 98 F T T <--- 3 060-079 56 56 56 87 87 87 F F F 4 080-099 62 66 66 99 107 107 F F F <--- 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 65 66 66 104 106 106 T T T <--- 9 180-199 65 66 66 104 106 106 T F F <--- 10 200-219 62 62 62 98 98 98 F F F 11 220-239 60 66 66 96 107 107 F F F <--- 12 240-259 56 56 56 88 88 88 F F F 13 260-279 65 66 66 104 107 107 F T T <--- 14 280-299 64 64 64 103 103 103 F T T <--- 15 300-319 65 66 66 104 106 106 F T T <--- 16 320-339 61 66 66 97 106 106 F F F <--- 17 340-359 61 66 66 97 107 107 F F F <--- 18 360-379 65 66 66 104 107 107 F F F <--- 19 380-399 65 65 65 104 104 104 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 64 66 66 103 106 106 F F F <--- 22 440-459 62 66 66 98 107 107 F F F <--- 23 460-479 62 62 62 98 98 98 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 63 63 63 100 100 100 F F F 26 520-539 63 63 63 100 100 100 F T T <--- 27 540-559 65 65 65 104 104 104 F F F 28 560-579 65 65 65 104 104 104 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 66, Max flush: 107 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 67 67 67 108 108 108 F F F 9 180-199 66 66 66 106 106 106 F F F 10 200-219 62 62 62 98 98 98 F F F 11 220-239 66 66 66 107 107 107 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 67 67 67 108 108 108 F F F 14 280-299 67 67 67 108 108 108 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 66 66 66 106 106 106 F F F 17 340-359 66 66 66 107 107 107 F F F 18 360-379 66 66 66 107 107 107 F F F 19 380-399 65 65 65 104 104 104 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 66 66 66 106 106 106 F F F 22 440-459 66 66 66 107 107 107 F F F 23 460-479 62 62 62 98 98 98 F F F 24 480-499 61 61 61 97 97 97 F F F 25 500-519 63 63 63 100 100 100 F F F 26 520-539 67 67 67 108 108 108 F F F 27 540-559 65 65 65 104 104 104 F F F 28 560-579 65 65 65 104 104 104 F F F 29 580-599 61 61 61 97 97 97 F F F Max gen: 67, Max flush: 108 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.094749907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.103494707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.10379014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.103813921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.103822961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.103834861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Crucible 68d1681c-7304-487c-905d-48f2fcc4d03e has session id: 498c7a8a-e56d-4c43-8464-9d638f1f8b2c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104250486Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Upstairs opts: Upstairs UUID: 68d1681c-7304-487c-905d-48f2fcc4d03e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104279656Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Crucible stats registered with UUID: 68d1681c-7304-487c-905d-48f2fcc4d03e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104290726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:46.104302066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104482778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104512659Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104523929Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.1046347Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.10465365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"68d1681c-7304-487c-905d-48f2fcc4d03e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104669111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104678631Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104688591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104698461Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104708261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104743401Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104764952Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43466)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104785102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58884)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104800212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34130)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.104820742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.10550612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.10552638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.10553773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105830114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105852744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[0]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 108, 106, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105865894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[0]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 67, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105877164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105888844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[1]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 108, 106, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105899985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[1]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 67, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105910675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105922075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[2]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 108, 106, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105932855Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[2]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 67, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105943475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105954945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105964815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105974975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105984546Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.105994266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"":"downstairs","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106004626Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106013886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"0","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106025176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"1","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106035526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"client":"2","":"downstairs","session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106046046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106055616Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"68d1681c-7304-487c-905d-48f2fcc4d03e is now active with session: 498c7a8a-e56d-4c43-8464-9d638f1f8b2c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106065897Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106074987Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"session_id":"498c7a8a-e56d-4c43-8464-9d638f1f8b2c"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106084557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Activated sub_volume 68d1681c-7304-487c-905d-48f2fcc4d03e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106092937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106101337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.106458371Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} 01/30 Write block 267 len 8192 data: 8 7 02/30 Read block 206 len 4096 03/30 Read block 288 len 12288 04/30 Write block 550 len 32768 data: 5 5 5 5 5 3 5 8 05/30 Write block 160 len 40960 data: 6 6 6 8 9 9 9 10 8 8 06/30 Read block 535 len 4096 07/30 Write block 177 len 20480 data: 7 9 9 11 9 08/30 Write block 351 len 20480 data: 5 4 4 6 5 09/30 Write block 428 len 24576 data: 5 6 10 8 8 9 10/30 Write block 221 len 28672 data: 4 5 5 5 5 5 6 11/30 Flush 12/30 Write block 549 len 16384 data: 5 6 6 6 13/30 Write block 498 len 16384 data: 7 8 9 9 14/30 Write block 594 len 12288 data: 3 3 3 15/30 Write block 356 len 28672 data: 5 5 6 6 7 6 4 16/30 Write block 445 len 36864 data: 5 5 4 5 5 7 5 4 5 17/30 Write block 474 len 24576 data: 5 4 4 4 5 4 18/30 Write block 370 len 8192 data: 4 4 19/30 Read block 472 len 12288 20/30 Write block 389 len 20480 data: 5 6 7 7 7 21/30 Read block 16 len 32768 22/30 Read block 328 len 40960 23/30 Write block 344 len 32768 data: 8 7 5 5 4 5 4 6 24/30 Read block 535 len 32768 25/30 Read block 320 len 12288 26/30 Write block 458 len 32768 data: 6 7 7 7 6 5 5 3 27/30 Read block 231 len 36864 28/30 Write block 228 len 4096 data: 6 29/30 Write block 538 len 32768 data: 9 7 8 9 7 7 8 9 30/30 Read block 516 len 28672 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 8 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 30 30 3 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:46.11421187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:46.114490683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1405,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 67 109 109 108 F F F <--- 9 180-199 68 68 66 109 109 106 F F F <--- 10 200-219 62 62 62 98 98 98 F F F 11 220-239 68 68 66 109 109 107 T T F <--- 12 240-259 56 56 56 88 88 88 F F F 13 260-279 68 68 67 109 109 108 F F T <--- 14 280-299 67 67 67 108 108 108 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 66 66 66 106 106 106 F F F 17 340-359 68 68 66 109 109 107 T T F <--- 18 360-379 66 66 66 107 107 107 T T F <--- 19 380-399 65 65 65 104 104 104 T T F <--- 20 400-419 61 61 61 97 97 97 F F F 21 420-439 68 68 66 109 109 106 F F F <--- 22 440-459 66 66 66 107 107 107 T T F <--- 23 460-479 62 62 62 98 98 98 T T F <--- 24 480-499 61 61 61 97 97 97 T T F <--- 25 500-519 63 63 63 100 100 100 T T F <--- 26 520-539 67 67 67 108 108 108 T T F <--- 27 540-559 68 68 65 109 109 104 T T F <--- 28 560-579 65 65 65 104 104 104 F F F 29 580-599 61 61 61 97 97 97 T T F <--- Max gen: 68, Max flush: 109 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 68 68 68 109 109 109 F F F 10 200-219 62 62 62 98 98 98 F F F 11 220-239 69 69 69 110 110 110 F F F 12 240-259 56 56 56 88 88 88 F F F 13 260-279 68 68 68 109 109 109 F F F 14 280-299 67 67 67 108 108 108 F F F 15 300-319 67 67 67 108 108 108 F F F 16 320-339 66 66 66 106 106 106 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 69 69 69 110 110 110 F F F 19 380-399 69 69 69 110 110 110 F F F 20 400-419 61 61 61 97 97 97 F F F 21 420-439 68 68 68 109 109 109 F F F 22 440-459 69 69 69 110 110 110 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 69 69 69 110 110 110 F F F 25 500-519 69 69 69 110 110 110 F F F 26 520-539 69 69 69 110 110 110 F F F 27 540-559 69 69 69 110 110 110 F F F 28 560-579 65 65 65 104 104 104 F F F 29 580-599 69 69 69 110 110 110 F F F Max gen: 69, Max flush: 110 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.36807622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.376486036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.376771129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.37679829Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.37680806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.37682023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Crucible 072dea30-4e95-4e54-9fdf-c9a745ffe9c2 has session id: c2205867-1286-42ca-b135-97a9520e1315","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377139233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Upstairs opts: Upstairs UUID: 072dea30-4e95-4e54-9fdf-c9a745ffe9c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377185064Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Crucible stats registered with UUID: 072dea30-4e95-4e54-9fdf-c9a745ffe9c2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377196264Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:50.377205804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377376046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377392506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377402927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377557618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377574668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"072dea30-4e95-4e54-9fdf-c9a745ffe9c2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377592399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377601759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377612099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377622359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377632519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377642959Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377655049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"ds_connection connected from Ok(127.0.0.1:52190)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.37773778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"ds_connection connected from Ok(127.0.0.1:46889)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377869052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"ds_connection connected from Ok(127.0.0.1:36907)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.377894802Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378359147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378435238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378471189Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378714712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378777942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[0]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 109, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378809463Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[0]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 68, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378821063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378832693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[1]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 109, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378843683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[1]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 68, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378854563Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378868333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[2]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 109, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378879433Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[2]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 68, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378890174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378901414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378911384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378921634Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378931424Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378941204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"":"downstairs","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378951664Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378961004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"0","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378971224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"1","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378981455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"client":"2","":"downstairs","session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.378991925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379001515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"072dea30-4e95-4e54-9fdf-c9a745ffe9c2 is now active with session: c2205867-1286-42ca-b135-97a9520e1315","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379012095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379022385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"session_id":"c2205867-1286-42ca-b135-97a9520e1315"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379031945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Activated sub_volume 072dea30-4e95-4e54-9fdf-c9a745ffe9c2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379040375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379048845Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.379395479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} 01/30 Write block 558 len 12288 data: 10 11 9 02/30 Read block 436 len 16384 03/30 Read block 513 len 28672 04/30 Write block 300 len 28672 data: 9 11 12 14 14 12 9 05/30 Write block 485 len 8192 data: 5 4 06/30 Read block 175 len 36864 07/30 Read block 395 len 20480 08/30 Write block 258 len 4096 data: 13 09/30 Flush 10/30 Write block 533 len 24576 data: 5 5 5 7 7 10 11/30 Write block 558 len 16384 data: 11 12 10 8 12/30 Read block 100 len 4096 13/30 Write block 482 len 40960 data: 4 6 6 6 5 4 3 4 6 6 14/30 Write block 190 len 28672 data: 7 9 11 11 11 9 10 15/30 Write block 293 len 4096 data: 6 16/30 Read block 290 len 28672 17/30 Read block 546 len 32768 18/30 Read block 69 len 16384 19/30 Read block 96 len 28672 20/30 Read block 471 len 36864 21/30 Read block 561 len 12288 22/30 Read block 489 len 24576 23/30 Read block 177 len 36864 24/30 Read block 458 len 24576 25/30 Read block 314 len 16384 26/30 Read block 359 len 8192 27/30 Read block 561 len 36864 28/30 Read block 454 len 16384 29/30 Write block 400 len 40960 data: 7 6 6 6 8 8 7 5 4 3 30/30 Read block 251 len 40960 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:50.387907867Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:50.38819422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1413,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 68 68 68 109 109 109 T F T <--- 10 200-219 62 62 62 98 98 98 F F F 11 220-239 69 69 69 110 110 110 F F F 12 240-259 70 56 70 111 88 111 F F F <--- 13 260-279 68 68 68 109 109 109 F F F 14 280-299 67 67 67 108 108 108 T F T <--- 15 300-319 70 67 70 111 108 111 F F F <--- 16 320-339 66 66 66 106 106 106 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 69 69 69 110 110 110 F F F 19 380-399 69 69 69 110 110 110 F F F 20 400-419 61 61 61 97 97 97 T F T <--- 21 420-439 68 68 68 109 109 109 F F F 22 440-459 69 69 69 110 110 110 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 70 69 70 111 110 111 T F T <--- 25 500-519 69 69 69 110 110 110 F F F 26 520-539 69 69 69 110 110 110 T F T <--- 27 540-559 70 69 70 111 110 111 T F T <--- 28 560-579 70 65 70 111 104 111 T F T <--- 29 580-599 69 69 69 110 110 110 F F F Max gen: 70, Max flush: 111 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 98 98 98 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 63 63 63 100 100 100 F F F 6 120-139 60 60 60 96 96 96 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 62 62 62 98 98 98 F F F 11 220-239 69 69 69 110 110 110 F F F 12 240-259 70 70 70 111 111 111 F F F 13 260-279 68 68 68 109 109 109 F F F 14 280-299 71 71 71 112 112 112 F F F 15 300-319 70 70 70 111 111 111 F F F 16 320-339 66 66 66 106 106 106 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 69 69 69 110 110 110 F F F 19 380-399 69 69 69 110 110 110 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 68 68 68 109 109 109 F F F 22 440-459 69 69 69 110 110 110 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 69 69 69 110 110 110 F F F 26 520-539 71 71 71 112 112 112 F F F 27 540-559 71 71 71 112 112 112 F F F 28 560-579 71 71 71 112 112 112 F F F 29 580-599 69 69 69 110 110 110 F F F Max gen: 71, Max flush: 112 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.622922061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63158357Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.631898844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.631924624Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.631933454Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.631945394Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Crucible 1063e226-6288-4eb3-bebb-948db521ae73 has session id: 4de44c73-9250-489c-837e-f1561a6ca6cb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632353989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Upstairs opts: Upstairs UUID: 1063e226-6288-4eb3-bebb-948db521ae73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632398179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Crucible stats registered with UUID: 1063e226-6288-4eb3-bebb-948db521ae73","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.6324093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:54.63241974Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632611612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632644342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632655882Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632780574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632805564Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"1063e226-6288-4eb3-bebb-948db521ae73 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632819764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632829194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632839494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632849685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632859795Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632877535Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632888035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:54605)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632944336Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45882)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.632970706Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52287)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633007976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633607573Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633638624Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633773435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633939497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.633949797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[0]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 112, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634113049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[0]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 71, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634124979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634136469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[1]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 112, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634147689Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[1]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 71, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63415852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63417026Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[2]R flush_numbers[0..12]: [98, 102, 108, 87, 107, 100, 96, 100, 109, 112, 98, 110]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63418124Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[2]R generation[0..12]: [62, 64, 67, 56, 66, 63, 60, 63, 68, 71, 62, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63419233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63420362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63421344Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63422351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.63423474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634244601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"":"downstairs","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634255271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634264751Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"0","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634275301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"1","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634285611Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"client":"2","":"downstairs","session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634296081Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634305891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"1063e226-6288-4eb3-bebb-948db521ae73 is now active with session: 4de44c73-9250-489c-837e-f1561a6ca6cb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634316421Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634325741Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"session_id":"4de44c73-9250-489c-837e-f1561a6ca6cb"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634335362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Activated sub_volume 1063e226-6288-4eb3-bebb-948db521ae73","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634343872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634352382Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.634773907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} 01/30 Write block 581 len 40960 data: 11 10 8 10 8 8 7 5 4 4 02/30 Read block 207 len 20480 03/30 Read block 486 len 8192 04/30 Flush 05/30 Flush 06/30 Write block 330 len 40960 data: 4 4 8 8 7 5 5 6 6 8 07/30 Read block 304 len 32768 08/30 Read block 124 len 8192 09/30 Read block 391 len 8192 10/30 Write block 109 len 4096 data: 9 11/30 Write block 232 len 12288 data: 6 6 5 12/30 Read block 367 len 24576 13/30 Write block 6 len 8192 data: 6 6 14/30 Write block 210 len 20480 data: 7 9 7 5 6 15/30 Write block 440 len 20480 data: 8 8 9 7 7 16/30 Write block 514 len 12288 data: 8 7 9 17/30 Read block 339 len 12288 18/30 Flush 19/30 Write block 369 len 16384 data: 6 5 5 3 20/30 Write block 133 len 20480 data: 11 11 7 5 4 21/30 Read block 325 len 4096 22/30 Read block 432 len 40960 23/30 Read block 558 len 4096 24/30 Write block 326 len 16384 data: 6 6 6 5 25/30 Write block 382 len 36864 data: 7 8 7 8 8 9 8 6 7 26/30 Read block 327 len 36864 27/30 Read block 515 len 40960 28/30 Write block 43 len 20480 data: 5 4 6 5 5 29/30 Write block 502 len 8192 data: 8 6 30/30 Write block 539 len 12288 data: 8 9 10 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 30 3 36 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:54.642864229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:54.643131902Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1421,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 62 72 115 98 115 F F F <--- 1 020-039 64 64 64 102 102 102 F F F 2 040-059 67 67 67 108 108 108 T F T <--- 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 72 63 72 115 100 115 F F F <--- 6 120-139 60 60 60 96 96 96 T F T <--- 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 72 62 72 115 98 115 F F F <--- 11 220-239 72 69 72 115 110 115 F F F <--- 12 240-259 70 70 70 111 111 111 F F F 13 260-279 68 68 68 109 109 109 F F F 14 280-299 71 71 71 112 112 112 F F F 15 300-319 70 70 70 111 111 111 F F F 16 320-339 72 66 72 115 106 115 T F T <--- 17 340-359 69 69 69 110 110 110 F F F 18 360-379 69 69 69 110 110 110 T F T <--- 19 380-399 69 69 69 110 110 110 T F T <--- 20 400-419 71 71 71 112 112 112 F F F 21 420-439 68 68 68 109 109 109 F F F 22 440-459 72 69 72 115 110 115 F F F <--- 23 460-479 69 69 69 110 110 110 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 72 69 72 115 110 115 T F T <--- 26 520-539 71 71 71 112 112 112 T F T <--- 27 540-559 71 71 71 112 112 112 T F T <--- 28 560-579 71 71 71 112 112 112 F F F 29 580-599 72 69 72 113 110 113 F F F <--- Max gen: 72, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 115 115 115 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 72 72 72 115 115 115 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 63 63 63 100 100 100 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 72 72 72 115 115 115 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 70 70 70 111 111 111 F F F 13 260-279 68 68 68 109 109 109 F F F 14 280-299 71 71 71 112 112 112 F F F 15 300-319 70 70 70 111 111 111 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 73 73 73 116 116 116 F F F 19 380-399 73 73 73 116 116 116 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 68 68 68 109 109 109 F F F 22 440-459 72 72 72 115 115 115 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 73 73 73 116 116 116 F F F 26 520-539 73 73 73 116 116 116 F F F 27 540-559 73 73 73 116 116 116 F F F 28 560-579 71 71 71 112 112 112 F F F 29 580-599 72 72 72 113 113 113 F F F Max gen: 73, Max flush: 116 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.891708221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:58.899061025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.899129926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90648068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.906736583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.906764813Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.906773773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.906785693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Crucible ccf24728-bd19-4607-b869-e27aae5e7034 has session id: d08ef4e8-259f-4ec4-9287-73cde52a811e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907157058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Upstairs opts: Upstairs UUID: ccf24728-bd19-4607-b869-e27aae5e7034, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907196758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Crucible stats registered with UUID: ccf24728-bd19-4607-b869-e27aae5e7034","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907207768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:58.907217308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907416701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907446671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907457871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907608033Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907623863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"ccf24728-bd19-4607-b869-e27aae5e7034 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907644503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907653923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907663943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907673904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907683914Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907694284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907742934Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41311)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907790475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61658)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907801745Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54881)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.907812485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908507923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908534543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908640035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908840277Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908850487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[0]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 115, 116, 100, 109, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908957318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[0]R generation[0..12]: [72, 64, 73, 56, 66, 72, 73, 63, 68, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908970878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908982999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[1]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 115, 116, 100, 109, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.908994179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[1]R generation[0..12]: [72, 64, 73, 56, 66, 72, 73, 63, 68, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909005219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909016719Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[2]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 115, 116, 100, 109, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909027999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[2]R generation[0..12]: [72, 64, 73, 56, 66, 72, 73, 63, 68, 71, 72, 72]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909038649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909050169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909060139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.9090703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90908103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90909159Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"":"downstairs","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90910176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90911107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"0","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90912229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"1","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90913242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"client":"2","":"downstairs","session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90914259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.90915213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msgDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"ccf24728-bd19-4607-b869-e27aae5e7034 is now active with session: d08ef4e8-259f-4ec4-9287-73cde52a811e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909166741Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909176161Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"session_id":"d08ef4e8-259f-4ec4-9287-73cde52a811e"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909185961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Activated sub_volume ccf24728-bd19-4607-b869-e27aae5e7034","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909194411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909203171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.909508125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} 01/30 Write block 287 len 24576 data: 4 5 6 6 5 6 02/30 Read block 171 len 16384 03/30 Read block 370 len 8192 04/30 Read block 5 len 12288 05/30 Write block 423 len 20480 data: 2 2 2 4 5 06/30 Flush 07/30 Write block 256 len 12288 data: 10 9 14 08/30 Write block 250 len 40960 data: 8 8 7 8 5 8 11 10 15 12 09/30 Write block 109 len 32768 data: 10 8 9 7 7 5 4 4 10/30 Read block 137 len 40960 11/30 Read block 457 len 28672 12/30 Write block 502 len 24576 data: 9 7 7 6 6 6 13/30 Read block 574 len 36864 14/30 Read block 197 len 12288 15/30 Read block 518 len 28672 16/30 Read block 441 len 32768 17/30 Write block 207 len 36864 data: 4 7 7 8 10 8 6 7 6 18/30 Read block 212 len 12288 19/30 Read block 293 len 16384 20/30 Read block 146 len 20480 21/30 Read block 103 len 16384 22/30 Write block 222 len 4096 data: 6 23/30 Read block 68 len 8192 24/30 Read block 237 len 28672 25/30 Flush 26/30 Write block 156 len 16384 data: 4 3 3 4 27/30 Write block 120 len 40960 data: 4 5 5 4 4 5 7 8 10 11 28/30 Write block 517 len 40960 data: 9 8 7 7 5 4 3 4 4 4 29/30 Write block 375 len 8192 data: 4 4 30/30 Read block 328 len 20480 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 17 0 1 18 Done 7 24 23 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:22:58.918804291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:22:58.919096044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1429,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 115 115 115 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 72 74 74 115 118 118 T F F <--- 6 120-139 73 73 73 116 116 116 F T T <--- 7 140-159 63 63 63 100 100 100 F T T <--- 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 72 74 74 115 118 118 F F F <--- 11 220-239 72 74 74 115 118 118 F F F <--- 12 240-259 70 74 74 111 118 118 T F F <--- 13 260-279 68 68 68 109 109 109 F F F 14 280-299 74 74 74 117 117 117 F F F 15 300-319 70 70 70 111 111 111 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 73 73 73 116 116 116 F T T <--- 19 380-399 73 73 73 116 116 116 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 72 72 72 115 115 115 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 73 74 74 116 118 118 T T T <--- 26 520-539 73 73 73 116 116 116 F T T <--- 27 540-559 73 73 73 116 116 116 F F F 28 560-579 71 71 71 112 112 112 F F F 29 580-599 72 72 72 113 113 113 F F F Max gen: 74, Max flush: 118 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 115 115 115 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 56 56 56 87 87 87 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 75 75 119 119 119 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 74 74 74 118 118 118 F F F 12 240-259 74 74 74 118 118 118 F F F 13 260-279 68 68 68 109 109 109 F F F 14 280-299 74 74 74 117 117 117 F F F 15 300-319 70 70 70 111 111 111 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 69 69 69 110 110 110 F F F 18 360-379 75 75 75 119 119 119 F F F 19 380-399 73 73 73 116 116 116 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 72 72 72 115 115 115 F F F 23 460-479 69 69 69 110 110 110 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 75 75 75 119 119 119 F F F 27 540-559 73 73 73 116 116 116 F F F 28 560-579 71 71 71 112 112 112 F F F 29 580-599 72 72 72 113 113 113 F F F Max gen: 75, Max flush: 119 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.165048424Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:03.17260188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.172677411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.179896134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180154927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180183187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180192227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180206857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Crucible b6d7b38c-5b59-417d-8d5e-156d91fa8dd6 has session id: 44685d26-e858-4d72-b165-fcb0eda51ab1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180577491Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Upstairs opts: Upstairs UUID: b6d7b38c-5b59-417d-8d5e-156d91fa8dd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180618252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Crucible stats registered with UUID: b6d7b38c-5b59-417d-8d5e-156d91fa8dd6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180630052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:03.180641542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180821524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180855425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.180866345Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181014696Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181040547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"b6d7b38c-5b59-417d-8d5e-156d91fa8dd6 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181049717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181059107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181069287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181079377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181089447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181101817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181112198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44396)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181156608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51574)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181167938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49118)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181178698Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181942247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181969007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.181981268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182286621Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182312551Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[0]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182369212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[0]R generation[0..12]: [72, 64, 73, 56, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182381112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182392732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[1]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182403882Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[1]R generation[0..12]: [72, 64, 73, 56, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182414692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182433583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[2]R flush_numbers[0..12]: [115, 102, 116, 87, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182444833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[2]R generation[0..12]: [72, 64, 73, 56, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182455533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182466973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182476923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182487153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182496943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182506814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"":"downstairs","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182517134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182526554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"0","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182536784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"1","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182546784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"client":"2","":"downstairs","session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182557334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182567084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"b6d7b38c-5b59-417d-8d5e-156d91fa8dd6 is now active with session: 44685d26-e858-4d72-b165-fcb0eda51ab1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182577604Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182588144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"session_id":"44685d26-e858-4d72-b165-fcb0eda51ab1"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182597835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Activated sub_volume b6d7b38c-5b59-417d-8d5e-156d91fa8dd6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182606395Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.182615115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.18303726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} 01/30 Write block 314 len 12288 data: 5 5 7 02/30 Read block 399 len 8192 03/30 Read block 37 len 24576 04/30 Write block 561 len 24576 data: 9 9 9 8 8 7 05/30 Read block 295 len 32768 06/30 Read block 34 len 20480 07/30 Write block 241 len 12288 data: 5 7 4 08/30 Read block 558 len 8192 09/30 Read block 359 len 28672 10/30 Read block 230 len 4096 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 346 len 4096 data: 6 15/30 Write block 70 len 20480 data: 6 6 6 5 5 16/30 Read block 138 len 40960 17/30 Write block 584 len 12288 data: 11 9 9 18/30 Write block 345 len 20480 data: 8 7 6 5 6 19/30 Write block 277 len 12288 data: 11 13 9 20/30 Read block 70 len 32768 21/30 Read block 403 len 8192 22/30 Read block 24 len 12288 23/30 Read block 176 len 32768 24/30 Write block 294 len 32768 data: 5 6 8 6 5 8 10 12 25/30 Read block 548 len 40960 26/30 Write block 389 len 40960 data: 7 8 8 8 8 6 6 5 6 8 27/30 Read block 293 len 12288 28/30 Read block 422 len 32768 29/30 Read block 90 len 32768 30/30 Write block 460 len 28672 data: 8 8 7 6 6 4 4 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:03.19178665Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:03.192058633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1437,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 115 115 115 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 56 56 56 87 87 87 F T T <--- 4 080-099 66 66 66 107 107 107 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 75 75 119 119 119 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 74 74 74 118 118 118 F F F 12 240-259 74 76 76 118 120 120 F F F <--- 13 260-279 68 68 68 109 109 109 F T T <--- 14 280-299 74 74 74 117 117 117 F T T <--- 15 300-319 70 76 76 111 120 120 F T T <--- 16 320-339 73 73 73 116 116 116 F F F 17 340-359 69 69 69 110 110 110 F T T <--- 18 360-379 75 75 75 119 119 119 F F F 19 380-399 73 73 73 116 116 116 F T T <--- 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 72 72 72 115 115 115 F F F 23 460-479 69 69 69 110 110 110 F T T <--- 24 480-499 71 71 71 112 112 112 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 75 75 75 119 119 119 F F F 27 540-559 73 73 73 116 116 116 F F F 28 560-579 71 76 76 112 120 120 F F F <--- 29 580-599 72 72 72 113 113 113 F T T <--- Max gen: 76, Max flush: 120 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 115 115 115 F F F 1 020-039 64 64 64 102 102 102 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 75 75 119 119 119 F F F 8 160-179 68 68 68 109 109 109 F F F 9 180-199 71 71 71 112 112 112 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 74 74 74 118 118 118 F F F 12 240-259 76 76 76 120 120 120 F F F 13 260-279 77 77 77 121 121 121 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 77 77 77 121 121 121 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 77 77 77 121 121 121 F F F 18 360-379 75 75 75 119 119 119 F F F 19 380-399 77 77 77 121 121 121 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 72 72 72 115 115 115 F F F 23 460-479 77 77 77 121 121 121 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 75 75 75 119 119 119 F F F 27 540-559 73 73 73 116 116 116 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 77, Max flush: 121 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.441363241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.450224002Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.450551736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.450577066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.450586136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.450598046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Crucible 2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c has session id: 1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45091509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Upstairs opts: Upstairs UUID: 2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45094537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Crucible stats registered with UUID: 2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45095675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:07.450966601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451133462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451149663Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451160263Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451325815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451341685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451357955Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451367295Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451377365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451387525Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451397526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451408056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451418496Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:52004)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451501827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37589)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451514197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48976)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.451525087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452239845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452265385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452303456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452548289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452573079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[0]R flush_numbers[0..12]: [115, 102, 116, 121, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452617029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[0]R generation[0..12]: [72, 64, 73, 77, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45262929Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.4526411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[1]R flush_numbers[0..12]: [115, 102, 116, 121, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45265297Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[1]R generation[0..12]: [72, 64, 73, 77, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45266381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45267513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[2]R flush_numbers[0..12]: [115, 102, 116, 121, 107, 118, 119, 119, 109, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.4526864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[2]R generation[0..12]: [72, 64, 73, 77, 66, 74, 75, 75, 68, 71, 74, 74]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.45269712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452708341Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452718331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452731421Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452741381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452751301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"":"downstairs","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452761481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452770781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"0","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452781011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"1","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452791221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"client":"2","":"downstairs","session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452801492Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452811032Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c is now active with session: 1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452821472Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452830612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"session_id":"1fdf9dbe-cb57-4ecf-ab1c-ad1cb9ca5dc7"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452840172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Activated sub_volume 2a0fa8c9-96c0-4bd4-9313-5fe8ca132f7c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452848722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.452857392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.453264757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} 01/30 Write block 225 len 28672 data: 6 6 7 7 6 5 6 02/30 Write block 546 len 28672 data: 11 10 6 6 7 7 7 03/30 Flush 04/30 Write block 190 len 40960 data: 8 10 12 12 12 10 11 8 7 6 05/30 Read block 16 len 36864 06/30 Read block 352 len 20480 07/30 Flush 08/30 Write block 531 len 24576 data: 4 4 6 6 6 8 09/30 Read block 137 len 40960 10/30 Read block 582 len 32768 11/30 Write block 371 len 16384 data: 6 4 2 4 12/30 Flush 13/30 Write block 230 len 40960 data: 6 7 7 7 6 6 6 5 6 6 14/30 Read block 400 len 28672 15/30 Flush 16/30 Write block 15 len 24576 data: 6 5 5 4 5 5 17/30 Write block 1 len 20480 data: 4 4 5 4 4 18/30 Write block 455 len 20480 data: 4 5 5 7 8 19/30 Write block 176 len 20480 data: 8 8 10 10 12 20/30 Read block 93 len 20480 21/30 Read block 402 len 12288 22/30 Read block 321 len 32768 23/30 Read block 402 len 16384 24/30 Flush 25/30 Write block 202 len 8192 data: 8 7 26/30 Read block 336 len 40960 27/30 Write block 1 len 4096 data: 5 28/30 Write block 37 len 36864 data: 4 3 3 2 3 3 6 5 7 29/30 Read block 218 len 4096 30/30 Write block 550 len 16384 data: 8 8 8 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:07.462436422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:07.462702465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1445,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 72 78 126 115 126 T F T <--- 1 020-039 78 64 78 126 102 126 T F T <--- 2 040-059 73 73 73 116 116 116 T F T <--- 3 060-079 77 77 77 121 121 121 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 75 75 119 119 119 F F F 8 160-179 78 68 78 126 109 126 F F F <--- 9 180-199 78 71 78 126 112 126 F F F <--- 10 200-219 74 74 74 118 118 118 T F T <--- 11 220-239 78 74 78 125 118 125 F T F <--- 12 240-259 76 76 76 120 120 120 F F F 13 260-279 77 77 77 121 121 121 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 77 77 77 121 121 121 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 77 77 77 121 121 121 F F F 18 360-379 78 75 78 124 119 124 F F F <--- 19 380-399 77 77 77 121 121 121 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 78 72 78 126 115 126 F F F <--- 23 460-479 77 77 77 121 121 121 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 78 75 78 124 119 124 F F F <--- 27 540-559 78 73 78 122 116 122 T F T <--- 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 78, Max flush: 126 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 66 66 66 107 107 107 F F F 5 100-119 74 74 74 118 118 118 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 75 75 119 119 119 F F F 8 160-179 78 78 78 126 126 126 F F F 9 180-199 78 78 78 126 126 126 F F F 10 200-219 79 79 79 127 127 127 F F F 11 220-239 78 78 78 125 125 125 F F F 12 240-259 76 76 76 120 120 120 F F F 13 260-279 77 77 77 121 121 121 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 77 77 77 121 121 121 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 77 77 77 121 121 121 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 77 77 77 121 121 121 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 78 78 78 126 126 126 F F F 23 460-479 77 77 77 121 121 121 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 78 78 78 124 124 124 F F F 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 79, Max flush: 127 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.709403313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:11.717166642Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.717242013Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.724780329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725048502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725073422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725082742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725095073Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Crucible 2abdf8a8-1930-4005-8599-80088d7f26e1 has session id: 2288a873-fe10-4661-a6e3-c92be59a578f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725494217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Upstairs opts: Upstairs UUID: 2abdf8a8-1930-4005-8599-80088d7f26e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725525487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Crucible stats registered with UUID: 2abdf8a8-1930-4005-8599-80088d7f26e1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725537418Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:11.725547518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.72574462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.72577544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.72578722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725925682Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725947432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"2abdf8a8-1930-4005-8599-80088d7f26e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725960622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725970313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725980823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.725991483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726012693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726023503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726034243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52076)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726049083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46192)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726060374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36812)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726079814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726739581Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726768322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.726892563Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727047435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727069585Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 127, 121, 107, 118, 119, 119, 126, 126, 127, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727186746Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 77, 66, 74, 75, 75, 78, 78, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727198827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727210717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 127, 121, 107, 118, 119, 119, 126, 126, 127, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727222247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 77, 66, 74, 75, 75, 78, 78, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727233427Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727245097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 127, 121, 107, 118, 119, 119, 126, 126, 127, 125]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727256457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 77, 66, 74, 75, 75, 78, 78, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727267457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727279118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727289248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727299848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727310218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727320498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"":"downstairs","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727331008Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727341918Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"0","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727352338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"1","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727362888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"client":"2","":"downstairs","session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727373669Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727383599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"2abdf8a8-1930-4005-8599-80088d7f26e1 is now active with session: 2288a873-fe10-4661-a6e3-c92be59a578f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727394389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727403949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"session_id":"2288a873-fe10-4661-a6e3-c92be59a578f"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727413939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Activated sub_volume 2abdf8a8-1930-4005-8599-80088d7f26e1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727423039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727432119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.727874584Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} 01/30 Flush 02/30 Write block 91 len 12288 data: 5 7 7 03/30 Read block 564 len 16384 04/30 Read block 320 len 8192 05/30 Write block 97 len 40960 data: 9 6 6 8 9 10 10 9 12 12 06/30 Read block 527 len 28672 07/30 Write block 493 len 4096 data: 5 08/30 Flush 09/30 Write block 266 len 32768 data: 10 9 8 6 5 6 6 9 10/30 Write block 157 len 24576 data: 4 4 5 7 7 7 11/30 Read block 417 len 20480 12/30 Read block 11 len 16384 13/30 Read block 405 len 20480 14/30 Write block 209 len 40960 data: 8 9 11 9 7 8 7 8 9 6 15/30 Write block 245 len 20480 data: 5 6 6 7 7 16/30 Flush 17/30 Write block 236 len 8192 data: 7 6 18/30 Write block 341 len 16384 data: 10 9 9 9 19/30 Read block 554 len 28672 20/30 Flush 21/30 Write block 112 len 24576 data: 8 8 6 5 5 4 22/30 Flush 23/30 Write block 160 len 28672 data: 8 8 8 9 10 10 10 24/30 Read block 533 len 4096 25/30 Flush 26/30 Write block 91 len 28672 data: 6 8 8 6 7 8 10 27/30 Read block 547 len 32768 28/30 Write block 210 len 24576 data: 10 12 10 8 9 8 29/30 Read block 260 len 20480 30/30 Write block 14 len 12288 data: 6 7 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 8 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 1 1 24 Done 0 21 21 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:11.737511775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:11.737809948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1453,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F T T <--- 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 80 80 80 129 129 129 F T T <--- 5 100-119 80 80 80 129 132 132 F F F <--- 6 120-139 75 75 75 119 119 119 F F F 7 140-159 75 80 80 119 130 130 F F F <--- 8 160-179 78 80 80 126 133 133 F F F <--- 9 180-199 78 78 78 126 126 126 F F F 10 200-219 79 80 80 127 130 130 F T T <--- 11 220-239 78 80 80 125 131 131 F F F <--- 12 240-259 76 80 80 120 130 130 F F F <--- 13 260-279 77 80 80 121 130 130 F F F <--- 14 280-299 77 77 77 121 121 121 F F F 15 300-319 77 77 77 121 121 121 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 77 80 80 121 131 131 F F F <--- 18 360-379 78 78 78 124 124 124 F F F 19 380-399 77 77 77 121 121 121 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 78 78 78 126 126 126 F F F 23 460-479 77 77 77 121 121 121 F F F 24 480-499 80 80 80 129 129 129 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 78 78 78 124 124 124 F F F 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 80, Max flush: 133 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 134 134 134 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 81 81 81 134 134 134 F F F 5 100-119 80 80 80 132 132 132 F F F 6 120-139 75 75 75 119 119 119 F F F 7 140-159 80 80 80 130 130 130 F F F 8 160-179 80 80 80 133 133 133 F F F 9 180-199 78 78 78 126 126 126 F F F 10 200-219 81 81 81 134 134 134 F F F 11 220-239 80 80 80 131 131 131 F F F 12 240-259 80 80 80 130 130 130 F F F 13 260-279 80 80 80 130 130 130 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 77 77 77 121 121 121 F F F 16 320-339 73 73 73 116 116 116 F F F 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 77 77 77 121 121 121 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 78 78 78 126 126 126 F F F 23 460-479 77 77 77 121 121 121 F F F 24 480-499 80 80 80 129 129 129 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 78 78 78 124 124 124 F F F 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 81, Max flush: 134 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:15.991408134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000260305Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000571379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000596349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000605429Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000617729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Crucible 79df0f7a-8730-4e4e-9b87-003aa9c52d3e has session id: 3a384482-e3db-494c-9073-79e4f4d8d05d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000948843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Upstairs opts: Upstairs UUID: 79df0f7a-8730-4e4e-9b87-003aa9c52d3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000981544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Crucible stats registered with UUID: 79df0f7a-8730-4e4e-9b87-003aa9c52d3e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.000993784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:16.001016874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001203556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001233747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001245317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001376748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001402578Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"79df0f7a-8730-4e4e-9b87-003aa9c52d3e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001414599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001424099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001434459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001444489Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001454839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001465389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001475869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51829)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.00154144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52722)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.00157926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60207)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.001591051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002295569Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002327239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.00243108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002691183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002701333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[0]R flush_numbers[0..12]: [134, 127, 127, 121, 134, 132, 119, 130, 133, 126, 134, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002810495Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[0]R generation[0..12]: [81, 79, 79, 77, 81, 80, 75, 80, 80, 78, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002822335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002833975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[1]R flush_numbers[0..12]: [134, 127, 127, 121, 134, 132, 119, 130, 133, 126, 134, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002845325Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[1]R generation[0..12]: [81, 79, 79, 77, 81, 80, 75, 80, 80, 78, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002856205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002867665Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[2]R flush_numbers[0..12]: [134, 127, 127, 121, 134, 132, 119, 130, 133, 126, 134, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002880665Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[2]R generation[0..12]: [81, 79, 79, 77, 81, 80, 75, 80, 80, 78, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002891535Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002902916Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002912946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002923206Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002933036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002942976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"":"downstairs","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002953246Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002962526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"0","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002972836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"1","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002983147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"client":"2","":"downstairs","session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.002993547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003003367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"79df0f7a-8730-4e4e-9b87-003aa9c52d3e is now active with session: 3a384482-e3db-494c-9073-79e4f4d8d05d","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003013927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003023147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"session_id":"3a384482-e3db-494c-9073-79e4f4d8d05d"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003034117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Activated sub_volume 79df0f7a-8730-4e4e-9b87-003aa9c52d3e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003042697Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003051287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.003433272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} 01/30 Flush 02/30 Write block 474 len 8192 data: 6 5 03/30 Write block 136 len 40960 data: 6 5 6 5 4 4 4 4 4 4 04/30 Write block 326 len 28672 data: 7 7 7 6 5 5 9 05/30 Read block 560 len 4096 06/30 Write block 6 len 12288 data: 7 7 5 07/30 Write block 218 len 12288 data: 7 5 5 08/30 Flush 09/30 Write block 239 len 8192 data: 7 4 10/30 Read block 380 len 24576 11/30 Write block 311 len 4096 data: 7 12/30 Flush 13/30 Write block 396 len 12288 data: 6 7 9 14/30 Read block 302 len 4096 15/30 Read block 119 len 28672 16/30 Write block 30 len 12288 data: 3 4 4 17/30 Read block 254 len 28672 18/30 Write block 12 len 4096 data: 6 19/30 Read block 134 len 12288 20/30 Read block 451 len 4096 21/30 Write block 165 len 12288 data: 11 11 11 22/30 Read block 247 len 8192 23/30 Read block 458 len 8192 24/30 Read block 217 len 36864 25/30 Read block 340 len 16384 26/30 Write block 212 len 20480 data: 11 9 10 9 9 27/30 Read block 537 len 20480 28/30 Write block 536 len 16384 data: 9 8 11 9 29/30 Write block 456 len 36864 data: 6 6 8 9 9 9 8 7 7 30/30 Write block 186 len 28672 data: 11 9 7 6 9 11 13 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Write 4 Done Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 24 3 29 Done 27 5 26 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1000)) Some(JobId(1011)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:16.010883187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:16.01118608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1461,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 81 82 136 134 136 T T T <--- 1 020-039 79 79 79 127 127 127 T F T <--- 2 040-059 79 79 79 127 127 127 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 81 81 81 134 134 134 F F F 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 75 82 136 119 136 F T F <--- 7 140-159 82 80 82 136 130 136 F T F <--- 8 160-179 80 80 80 133 133 133 T F T <--- 9 180-199 78 78 78 126 126 126 T F T <--- 10 200-219 82 81 82 136 134 136 T F T <--- 11 220-239 82 80 82 137 131 137 F F F <--- 12 240-259 82 80 82 137 130 137 F F F <--- 13 260-279 80 80 80 130 130 130 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 82 77 82 137 121 137 F F F <--- 16 320-339 82 73 82 136 116 136 F T F <--- 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 77 77 77 121 121 121 T F T <--- 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 78 78 78 126 126 126 T F T <--- 23 460-479 82 77 82 136 121 136 T T T <--- 24 480-499 80 80 80 129 129 129 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 78 78 78 124 124 124 T F T <--- 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 82, Max flush: 137 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 138 138 138 F F F 1 020-039 83 83 83 138 138 138 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 81 81 81 134 134 134 F F F 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 82 82 136 136 136 F F F 7 140-159 82 82 82 136 136 136 F F F 8 160-179 83 83 83 138 138 138 F F F 9 180-199 83 83 83 138 138 138 F F F 10 200-219 83 83 83 138 138 138 F F F 11 220-239 82 82 82 137 137 137 F F F 12 240-259 82 82 82 137 137 137 F F F 13 260-279 80 80 80 130 130 130 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 82 82 82 137 137 137 F F F 16 320-339 82 82 82 136 136 136 F F F 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 83 83 83 138 138 138 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 83 83 83 138 138 138 F F F 23 460-479 83 83 83 138 138 138 F F F 24 480-499 80 80 80 129 129 129 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 83 83 83 138 138 138 F F F 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 77 77 77 121 121 121 F F F Max gen: 83, Max flush: 138 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.269945466Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.278449094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.278758807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.278796478Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.278805878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.278817758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Crucible 28b9278a-7c93-4655-a898-368d79b29196 has session id: a66d21d0-7912-4b9f-8189-e9b59d252c47","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279212313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Upstairs opts: Upstairs UUID: 28b9278a-7c93-4655-a898-368d79b29196, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279251873Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Crucible stats registered with UUID: 28b9278a-7c93-4655-a898-368d79b29196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279262853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:20.279272803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279452635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279474756Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279485216Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279661798Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279685978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"28b9278a-7c93-4655-a898-368d79b29196 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279702218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279711518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279721648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279731818Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279741919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279752379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279788099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"ds_connection connected from Ok(127.0.0.1:53884)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279813439Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"ds_connection connected from Ok(127.0.0.1:45721)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.27982515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"ds_connection connected from Ok(127.0.0.1:41863)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.279965651Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280504887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280519287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280563228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280830281Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280840191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[0]R flush_numbers[0..12]: [138, 138, 127, 121, 134, 132, 136, 136, 138, 138, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280875422Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[0]R generation[0..12]: [83, 83, 79, 77, 81, 80, 82, 82, 83, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280886872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280898312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[1]R flush_numbers[0..12]: [138, 138, 127, 121, 134, 132, 136, 136, 138, 138, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280909372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[1]R generation[0..12]: [83, 83, 79, 77, 81, 80, 82, 82, 83, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280920202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280931452Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[2]R flush_numbers[0..12]: [138, 138, 127, 121, 134, 132, 136, 136, 138, 138, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280942512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[2]R generation[0..12]: [83, 83, 79, 77, 81, 80, 82, 82, 83, 83, 83, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280953252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280964553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280974263Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280984223Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.280995503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281005323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"":"downstairs","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281015373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281024523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"0","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281034493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"1","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281044513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"client":"2","":"downstairs","session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281054664Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281064194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"28b9278a-7c93-4655-a898-368d79b29196 is now active with session: a66d21d0-7912-4b9f-8189-e9b59d252c47","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281074504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281083564Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"session_id":"a66d21d0-7912-4b9f-8189-e9b59d252c47"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281093074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Activated sub_volume 28b9278a-7c93-4655-a898-368d79b29196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281101834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281110364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.281524219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} 01/30 Write block 497 len 4096 data: 6 02/30 Write block 16 len 4096 data: 7 03/30 Flush 04/30 Write block 498 len 16384 data: 8 9 10 10 05/30 Write block 330 len 36864 data: 6 6 10 9 8 6 6 7 7 06/30 Read block 234 len 16384 07/30 Write block 39 len 36864 data: 4 3 4 4 7 6 8 6 6 08/30 Write block 459 len 20480 data: 10 10 10 9 8 09/30 Read block 359 len 32768 10/30 Read block 424 len 12288 11/30 Read block 493 len 12288 12/30 Write block 587 len 36864 data: 8 6 5 5 4 4 4 4 4 13/30 Write block 446 len 28672 data: 6 5 6 6 8 6 5 14/30 Write block 498 len 8192 data: 9 10 15/30 Read block 103 len 12288 16/30 Read block 60 len 16384 17/30 Write block 207 len 16384 data: 5 8 9 11 18/30 Write block 553 len 28672 data: 7 6 4 6 9 12 13 19/30 Write block 81 len 32768 data: 7 8 4 5 6 6 6 6 20/30 Read block 454 len 16384 21/30 Read block 404 len 40960 22/30 Write block 259 len 40960 data: 13 11 11 14 15 11 13 11 10 9 23/30 Write block 9 len 24576 data: 5 6 7 7 6 7 24/30 Flush 25/30 Write block 381 len 36864 data: 4 8 9 8 9 9 10 9 8 26/30 Write block 391 len 4096 data: 9 27/30 Read block 574 len 36864 28/30 Read block 271 len 40960 29/30 Write block 176 len 24576 data: 9 9 11 11 13 10 30/30 Read block 508 len 20480 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 2 31 Done 30 1 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:20.290432191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:20.290730384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1469,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 83 84 140 138 140 F F F <--- 1 020-039 84 83 84 140 138 140 F F F <--- 2 040-059 84 79 84 140 127 140 F F F <--- 3 060-079 77 77 77 121 121 121 F F F 4 080-099 84 81 84 140 134 140 F F F <--- 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 82 82 136 136 136 F F F 7 140-159 82 82 82 136 136 136 F F F 8 160-179 83 83 83 138 138 138 T F T <--- 9 180-199 83 83 83 138 138 138 T F T <--- 10 200-219 84 83 84 140 138 140 F F F <--- 11 220-239 82 82 82 137 137 137 F F F 12 240-259 84 82 84 140 137 140 F F F <--- 13 260-279 84 80 84 140 130 140 F F F <--- 14 280-299 77 77 77 121 121 121 F F F 15 300-319 82 82 82 137 137 137 F F F 16 320-339 84 82 84 140 136 140 F F F <--- 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 83 83 83 138 138 138 T F T <--- 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 84 83 84 140 138 140 F F F <--- 23 460-479 84 83 84 140 138 140 F F F <--- 24 480-499 84 80 84 140 129 140 F T F <--- 25 500-519 84 75 84 140 119 140 F F F <--- 26 520-539 83 83 83 138 138 138 F F F 27 540-559 84 79 84 140 127 140 F F F <--- 28 560-579 76 76 76 120 120 120 F F F 29 580-599 84 77 84 140 121 140 F F F <--- Max gen: 84, Max flush: 140 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 140 140 140 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 84 84 84 140 140 140 F F F 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 82 82 136 136 136 F F F 7 140-159 82 82 82 136 136 136 F F F 8 160-179 85 85 85 141 141 141 F F F 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 F F F 11 220-239 82 82 82 137 137 137 F F F 12 240-259 84 84 84 140 140 140 F F F 13 260-279 84 84 84 140 140 140 F F F 14 280-299 77 77 77 121 121 121 F F F 15 300-319 82 82 82 137 137 137 F F F 16 320-339 84 84 84 140 140 140 F F F 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 71 71 71 112 112 112 F F F 21 420-439 74 74 74 117 117 117 F F F 22 440-459 84 84 84 140 140 140 F F F 23 460-479 84 84 84 140 140 140 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 84 84 84 140 140 140 F F F 26 520-539 83 83 83 138 138 138 F F F 27 540-559 84 84 84 140 140 140 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 85, Max flush: 141 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.542870675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.551685726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552025759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55205124Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55206109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55207339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Crucible 5bfd57f8-af1c-4ec7-8189-38fbaa38e89e has session id: d266e33c-bc39-4ab5-90ce-e7f3fb23469e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552432494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Upstairs opts: Upstairs UUID: 5bfd57f8-af1c-4ec7-8189-38fbaa38e89e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552477055Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Crucible stats registered with UUID: 5bfd57f8-af1c-4ec7-8189-38fbaa38e89e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552488265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:24.552497945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552688437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552704717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552715457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552837619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552862259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"5bfd57f8-af1c-4ec7-8189-38fbaa38e89e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552879229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552888679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.552900289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55291051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55292088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55293212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.5529894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57670)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.553001461Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64581)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.553012411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43427)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.553118862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55378195Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.55381128Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.553942311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554141414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554164504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 140, 121, 140, 132, 136, 136, 141, 141, 140, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554266075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[0]R generation[0..12]: [84, 84, 84, 77, 84, 80, 82, 82, 85, 85, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554278175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554292145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 140, 121, 140, 132, 136, 136, 141, 141, 140, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554303425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[1]R generation[0..12]: [84, 84, 84, 77, 84, 80, 82, 82, 85, 85, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554314296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554325726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 140, 121, 140, 132, 136, 136, 141, 141, 140, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554336726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[2]R generation[0..12]: [84, 84, 84, 77, 84, 80, 82, 82, 85, 85, 84, 82]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554347406Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554358736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554368536Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554378576Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554388396Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554398187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"":"downstairs","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554408387Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554417687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"0","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554427967Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"1","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554439387Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"client":"2","":"downstairs","session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554449537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554459117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"5bfd57f8-af1c-4ec7-8189-38fbaa38e89e is now active with session: d266e33c-bc39-4ab5-90ce-e7f3fb23469e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554469437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554478677Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"session_id":"d266e33c-bc39-4ab5-90ce-e7f3fb23469e"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554493738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Activated sub_volume 5bfd57f8-af1c-4ec7-8189-38fbaa38e89e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554502288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554510878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.554928203Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} 01/30 Write block 259 len 24576 data: 14 12 12 15 16 12 02/30 Write block 459 len 8192 data: 11 11 03/30 Read block 431 len 4096 04/30 Write block 415 len 24576 data: 7 7 5 4 4 3 05/30 Read block 57 len 36864 06/30 Read block 139 len 12288 07/30 Write block 415 len 24576 data: 8 8 6 5 5 4 08/30 Read block 78 len 4096 09/30 Read block 457 len 40960 10/30 Write block 159 len 8192 data: 6 9 11/30 Read block 64 len 16384 12/30 Write block 95 len 12288 data: 8 9 11 13/30 Flush 14/30 Write block 327 len 20480 data: 8 8 7 7 7 15/30 Read block 473 len 24576 16/30 Write block 292 len 28672 data: 7 7 6 7 9 7 6 17/30 Read block 296 len 16384 18/30 Read block 404 len 4096 19/30 Write block 229 len 16384 data: 7 7 8 8 20/30 Read block 157 len 28672 21/30 Read block 579 len 40960 22/30 Write block 510 len 36864 data: 8 9 9 11 9 8 10 10 9 23/30 Read block 341 len 4096 24/30 Write block 528 len 4096 data: 4 25/30 Read block 489 len 4096 26/30 Read block 287 len 20480 27/30 Write block 547 len 20480 data: 11 7 7 9 9 28/30 Read block 114 len 8192 29/30 Write block 237 len 12288 data: 7 7 8 30/30 Read block 274 len 8192 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Write 2 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 30 32 Done 28 30 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:24.563191317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:24.56344276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1477,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 140 140 140 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 86 86 84 142 142 140 F F F <--- 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 82 82 136 136 136 F F F 7 140-159 86 86 82 142 142 136 F F F <--- 8 160-179 86 86 85 142 142 141 F F F <--- 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 F F F 11 220-239 82 82 82 137 137 137 T T F <--- 12 240-259 86 86 84 142 142 140 F F F <--- 13 260-279 86 86 84 142 142 140 F F F <--- 14 280-299 77 77 77 121 121 121 T T F <--- 15 300-319 82 82 82 137 137 137 F F F 16 320-339 84 84 84 140 140 140 T T F <--- 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 71 142 142 112 F F F <--- 21 420-439 86 86 74 142 142 117 F F F <--- 22 440-459 86 86 84 142 142 140 F F F <--- 23 460-479 86 86 84 142 142 140 F F F <--- 24 480-499 84 84 84 140 140 140 F F F 25 500-519 84 84 84 140 140 140 T T F <--- 26 520-539 83 83 83 138 138 138 T T F <--- 27 540-559 84 84 84 140 140 140 T T F <--- 28 560-579 76 76 76 120 120 120 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 86, Max flush: 142 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 140 140 140 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 86 86 86 142 142 142 F F F 5 100-119 80 80 80 132 132 132 F F F 6 120-139 82 82 82 136 136 136 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 86 86 86 142 142 142 F F F 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 F F F 11 220-239 87 87 87 143 143 143 F F F 12 240-259 86 86 86 142 142 142 F F F 13 260-279 86 86 86 142 142 142 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 82 82 82 137 137 137 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 80 80 80 131 131 131 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 86 86 86 142 142 142 F F F 22 440-459 86 86 86 142 142 142 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 87 87 87 143 143 143 F F F 26 520-539 87 87 87 143 143 143 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 76 76 76 120 120 120 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 87, Max flush: 143 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.81027293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.818734926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819030059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819066889Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819076579Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.8190889Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Crucible 69433a78-214b-4e53-876d-4525b62c7445 has session id: 730d0c89-cb1d-4276-a21c-711c0ac51e8e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819428283Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Upstairs opts: Upstairs UUID: 69433a78-214b-4e53-876d-4525b62c7445, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819466414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Crucible stats registered with UUID: 69433a78-214b-4e53-876d-4525b62c7445","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819477614Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:28.819487244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819677186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819706757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819718037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819840298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819854098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"69433a78-214b-4e53-876d-4525b62c7445 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819866358Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819875399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819885419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819895369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819905469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819915849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.819926099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45377)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.81997576Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44067)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.81998743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56942)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820089331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820641437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820661948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820740688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820965601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.820976741Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 140, 121, 142, 132, 136, 142, 142, 141, 140, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821017882Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[0]R generation[0..12]: [84, 84, 84, 77, 86, 80, 82, 86, 86, 85, 84, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821029692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821041222Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 140, 121, 142, 132, 136, 142, 142, 141, 140, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821052332Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[1]R generation[0..12]: [84, 84, 84, 77, 86, 80, 82, 86, 86, 85, 84, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821063272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821074582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 140, 121, 142, 132, 136, 142, 142, 141, 140, 143]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821087632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[2]R generation[0..12]: [84, 84, 84, 77, 86, 80, 82, 86, 86, 85, 84, 87]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821098453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821109513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821119333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821129353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821139073Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821148883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"":"downstairs","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821158853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821168043Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"0","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821178133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"1","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821188104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"client":"2","":"downstairs","session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821198144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821207724Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"69433a78-214b-4e53-876d-4525b62c7445 is now active with session: 730d0c89-cb1d-4276-a21c-711c0ac51e8e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821217934Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821226934Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"session_id":"730d0c89-cb1d-4276-a21c-711c0ac51e8e"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821236214Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Activated sub_volume 69433a78-214b-4e53-876d-4525b62c7445","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821245904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821254234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.821641329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 171 len 4096 data: 6 04/30 Flush 05/30 Write block 94 len 24576 data: 7 9 10 12 7 7 06/30 Write block 565 len 32768 data: 9 8 7 8 4 4 5 7 07/30 Read block 282 len 28672 08/30 Write block 315 len 16384 data: 6 8 8 7 09/30 Read block 387 len 4096 10/30 Write block 225 len 24576 data: 7 7 8 8 8 8 11/30 Write block 520 len 40960 data: 8 6 5 4 5 5 5 4 5 4 12/30 Read block 108 len 16384 13/30 Write block 17 len 4096 data: 6 14/30 Write block 265 len 12288 data: 14 12 11 15/30 Write block 169 len 28672 data: 9 7 7 6 8 8 8 16/30 Read block 70 len 28672 17/30 Read block 200 len 8192 18/30 Read block 247 len 12288 19/30 Read block 318 len 8192 20/30 Read block 108 len 28672 21/30 Read block 388 len 32768 22/30 Read block 585 len 40960 23/30 Write block 258 len 4096 data: 16 24/30 Flush 25/30 Flush 26/30 Write block 343 len 16384 data: 10 10 9 8 27/30 Read block 208 len 12288 28/30 Write block 98 len 36864 data: 8 8 9 10 11 11 10 13 13 29/30 Read block 459 len 16384 30/30 Write block 136 len 16384 data: 7 6 7 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 22 24 Done 25 25 4 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:28.829703271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:28.829984884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1485,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 84 147 147 140 F F F <--- 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 88 88 86 147 147 142 T T T <--- 5 100-119 80 80 80 132 132 132 T T F <--- 6 120-139 82 82 82 136 136 136 T T F <--- 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 146 F F F <--- 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 F F F 11 220-239 88 88 87 147 147 143 F F F <--- 12 240-259 88 88 86 147 147 142 F F F <--- 13 260-279 88 88 86 147 147 142 F F F <--- 14 280-299 87 87 87 143 143 143 F F F 15 300-319 88 88 82 147 147 137 F F T <--- 16 320-339 87 87 87 143 143 143 F F F 17 340-359 80 80 80 131 131 131 T T F <--- 18 360-379 78 78 78 124 124 124 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 86 86 86 142 142 142 F F F 22 440-459 86 86 86 142 142 142 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 87 87 87 143 143 143 F F F 26 520-539 88 88 87 147 147 143 F F F <--- 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 76 147 147 120 F F T <--- 29 580-599 84 84 84 140 140 140 F F F Max gen: 88, Max flush: 147 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 147 147 147 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 89 89 89 148 148 148 F F F 5 100-119 89 89 89 148 148 148 F F F 6 120-139 89 89 89 148 148 148 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 F F F 11 220-239 88 88 88 147 147 147 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 88 88 88 147 147 147 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 88 88 88 147 147 147 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 89 89 89 148 148 148 F F F 18 360-379 78 78 78 124 124 124 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 86 86 86 142 142 142 F F F 22 440-459 86 86 86 142 142 142 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 84 84 84 140 140 140 F F F 25 500-519 87 87 87 143 143 143 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 84 84 84 140 140 140 F F F Max gen: 89, Max flush: 148 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.075747441Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.084256119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.084560022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.084584563Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.084593743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.084606153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Crucible 2975824d-f7f9-4999-b437-57fe4f0b9812 has session id: 72d2124f-f1f4-4a96-9031-44dbbd9899c7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085009337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Upstairs opts: Upstairs UUID: 2975824d-f7f9-4999-b437-57fe4f0b9812, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085044258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Crucible stats registered with UUID: 2975824d-f7f9-4999-b437-57fe4f0b9812","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085055608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:33.085065338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08525866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085289091Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085303051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085438772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085457713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"2975824d-f7f9-4999-b437-57fe4f0b9812 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085474673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085484363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085494663Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085505023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085515153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085525853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085536524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62382)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085611714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49435)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085623315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49163)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.085634105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086371023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086392103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086425814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086717777Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086755977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[0]R flush_numbers[0..12]: [147, 140, 140, 121, 148, 148, 148, 142, 147, 141, 140, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086806888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[0]R generation[0..12]: [88, 84, 84, 77, 89, 89, 89, 86, 88, 85, 84, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086818338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086829818Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[1]R flush_numbers[0..12]: [147, 140, 140, 121, 148, 148, 148, 142, 147, 141, 140, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086840858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[1]R generation[0..12]: [88, 84, 84, 77, 89, 89, 89, 86, 88, 85, 84, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086851599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086862989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[2]R flush_numbers[0..12]: [147, 140, 140, 121, 148, 148, 148, 142, 147, 141, 140, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086873979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[2]R generation[0..12]: [88, 84, 84, 77, 89, 89, 89, 86, 88, 85, 84, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086884729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086895959Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086906009Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086916069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.086925859Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08693762Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"":"downstairs","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08694784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08695709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"0","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.0869671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"1","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08697803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"client":"2","":"downstairs","session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08698824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08699789Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"2975824d-f7f9-4999-b437-57fe4f0b9812 is now active with session: 72d2124f-f1f4-4a96-9031-44dbbd9899c7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08700834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.08701758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"session_id":"72d2124f-f1f4-4a96-9031-44dbbd9899c7"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.087026951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Activated sub_volume 2975824d-f7f9-4999-b437-57fe4f0b9812","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.087035301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.087043721Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.087417405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} 01/30 Write block 514 len 4096 data: 10 02/30 Read block 490 len 28672 03/30 Flush 04/30 Write block 452 len 4096 data: 6 05/30 Write block 313 len 8192 data: 7 6 06/30 Read block 402 len 40960 07/30 Write block 98 len 24576 data: 9 9 10 11 12 12 08/30 Flush 09/30 Write block 53 len 28672 data: 7 6 7 7 5 6 8 10/30 Read block 492 len 8192 11/30 Read block 491 len 4096 12/30 Read block 531 len 8192 13/30 Read block 415 len 36864 14/30 Write block 498 len 24576 data: 10 11 11 11 10 8 15/30 Read block 244 len 20480 16/30 Read block 68 len 28672 17/30 Read block 204 len 4096 18/30 Read block 320 len 32768 19/30 Read block 266 len 8192 20/30 Read block 199 len 20480 21/30 Read block 149 len 24576 22/30 Write block 200 len 4096 data: 5 23/30 Write block 92 len 4096 data: 9 24/30 Read block 254 len 4096 25/30 Read block 296 len 4096 26/30 Write block 439 len 32768 data: 11 9 9 10 8 8 6 7 27/30 Write block 588 len 24576 data: 7 6 6 5 5 5 28/30 Write block 119 len 36864 data: 4 5 6 6 5 5 6 8 9 29/30 Read block 523 len 12288 30/30 Write block 357 len 28672 data: 6 7 7 8 7 5 4 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 26 26 0 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:33.095378166Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:33.095657779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1493,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 147 147 147 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 84 84 84 140 140 140 T T F <--- 3 060-079 77 77 77 121 121 121 F F F 4 080-099 90 90 89 150 150 148 T T F <--- 5 100-119 90 90 89 150 150 148 T T F <--- 6 120-139 89 89 89 148 148 148 T T F <--- 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 85 85 85 141 141 141 F F F 10 200-219 84 84 84 140 140 140 T T F <--- 11 220-239 88 88 88 147 147 147 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 88 88 88 147 147 147 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 88 150 150 147 F F F <--- 16 320-339 87 87 87 143 143 143 F F F 17 340-359 89 89 89 148 148 148 T T F <--- 18 360-379 78 78 78 124 124 124 T T F <--- 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 86 86 86 142 142 142 T T F <--- 22 440-459 90 90 86 150 150 142 T T F <--- 23 460-479 86 86 86 142 142 142 F F F 24 480-499 84 84 84 140 140 140 T T F <--- 25 500-519 90 90 90 149 149 149 T T F <--- 26 520-539 88 88 88 147 147 147 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 84 84 84 140 140 140 T T F <--- Max gen: 90, Max flush: 150 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 147 147 147 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 91 91 91 151 151 151 F F F 3 060-079 77 77 77 121 121 121 F F F 4 080-099 91 91 91 151 151 151 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 85 85 85 141 141 141 F F F 10 200-219 91 91 91 151 151 151 F F F 11 220-239 88 88 88 147 147 147 F F F 12 240-259 88 88 88 147 147 147 F F F 13 260-279 88 88 88 147 147 147 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 91 91 91 151 151 151 F F F 19 380-399 85 85 85 141 141 141 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 91 91 91 151 151 151 F F F 22 440-459 91 91 91 151 151 151 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 91 91 91 151 151 151 F F F 25 500-519 91 91 91 151 151 151 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 91 91 91 151 151 151 F F F Max gen: 91, Max flush: 151 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.345137079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.353759428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354062041Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354086492Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354097882Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354109792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Crucible c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1 has session id: 0cb22836-da31-462e-af0a-c12233455372","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354507606Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Upstairs opts: Upstairs UUID: c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354557507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Crucible stats registered with UUID: c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354568607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:37.354578127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354761739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.35477706Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.35478828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354890361Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354914771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354938381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354948081Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354958152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354968212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354979722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.354990192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355000342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"ds_connection connected from Ok(127.0.0.1:38296)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355076463Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"ds_connection connected from Ok(127.0.0.1:57928)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355102593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"ds_connection connected from Ok(127.0.0.1:56940)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355205284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355817281Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355841702Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.355853382Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356146395Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356182696Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[0]R flush_numbers[0..12]: [147, 140, 151, 121, 151, 151, 151, 142, 147, 141, 151, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356235206Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[0]R generation[0..12]: [88, 84, 91, 77, 91, 91, 91, 86, 88, 85, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356246766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356258286Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[1]R flush_numbers[0..12]: [147, 140, 151, 121, 151, 151, 151, 142, 147, 141, 151, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356269307Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[1]R generation[0..12]: [88, 84, 91, 77, 91, 91, 91, 86, 88, 85, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356281957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356293217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[2]R flush_numbers[0..12]: [147, 140, 151, 121, 151, 151, 151, 142, 147, 141, 151, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356304087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[2]R generation[0..12]: [88, 84, 91, 77, 91, 91, 91, 86, 88, 85, 91, 88]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356314727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356325927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356335737Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356345837Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356355548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356365368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"":"downstairs","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356375518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356384728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"0","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356394868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"1","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356404648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"client":"2","":"downstairs","session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356414778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356424278Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1 is now active with session: 0cb22836-da31-462e-af0a-c12233455372","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356436168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356445349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"session_id":"0cb22836-da31-462e-af0a-c12233455372"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356454839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Activated sub_volume c5402602-83a8-4b3d-ba2e-ad6eafa9d4a1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356463499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356471969Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.356910024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} 01/30 Write block 384 len 32768 data: 9 10 10 11 10 9 9 10 02/30 Read block 546 len 36864 03/30 Read block 329 len 40960 04/30 Read block 233 len 32768 05/30 Read block 415 len 32768 06/30 Read block 570 len 16384 07/30 Write block 65 len 28672 data: 7 7 7 6 5 7 7 08/30 Read block 299 len 20480 09/30 Flush 10/30 Flush 11/30 Write block 55 len 20480 data: 8 8 6 7 9 12/30 Read block 355 len 20480 13/30 Write block 79 len 36864 data: 5 7 8 9 5 6 7 7 7 14/30 Read block 157 len 8192 15/30 Read block 32 len 8192 16/30 Write block 195 len 36864 data: 11 12 9 8 7 6 7 9 8 17/30 Write block 14 len 20480 data: 8 8 8 7 5 18/30 Write block 182 len 12288 data: 8 8 7 19/30 Read block 80 len 12288 20/30 Write block 222 len 32768 data: 7 6 6 8 8 9 9 9 21/30 Read block 465 len 4096 22/30 Flush 23/30 Write block 431 len 12288 data: 9 9 10 24/30 Read block 507 len 40960 25/30 Read block 19 len 28672 26/30 Write block 9 len 24576 data: 6 7 8 8 7 9 27/30 Write block 378 len 12288 data: 4 4 4 28/30 Write block 367 len 20480 data: 5 4 7 6 7 29/30 Write block 241 len 36864 data: 6 8 5 4 6 7 7 8 8 30/30 Read block 62 len 4096 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 25 25 Done 30 30 5 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:37.365943447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:37.366237841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1501,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 88 154 154 147 T T F <--- 1 020-039 84 84 84 140 140 140 F F F 2 040-059 92 92 91 154 154 151 F F F <--- 3 060-079 92 92 77 154 154 121 F F F <--- 4 080-099 92 92 91 154 154 151 F F F <--- 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 85 154 154 141 F F F <--- 10 200-219 92 92 91 154 154 151 F F F <--- 11 220-239 92 92 88 154 154 147 F F F <--- 12 240-259 88 88 88 147 147 147 T T F <--- 13 260-279 88 88 88 147 147 147 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 91 91 91 151 151 151 T T F <--- 19 380-399 92 92 85 152 152 141 T T T <--- 20 400-419 86 86 86 142 142 142 F F F 21 420-439 91 91 91 151 151 151 T T F <--- 22 440-459 91 91 91 151 151 151 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 91 91 91 151 151 151 F F F 25 500-519 91 91 91 151 151 151 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 91 91 91 151 151 151 F F F Max gen: 92, Max flush: 154 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 155 155 155 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 92 92 92 154 154 154 F F F 4 080-099 92 92 92 154 154 154 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 92 92 92 154 154 154 F F F 11 220-239 92 92 92 154 154 154 F F F 12 240-259 93 93 93 155 155 155 F F F 13 260-279 88 88 88 147 147 147 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 91 91 91 151 151 151 F F F 18 360-379 93 93 93 155 155 155 F F F 19 380-399 93 93 93 155 155 155 F F F 20 400-419 86 86 86 142 142 142 F F F 21 420-439 93 93 93 155 155 155 F F F 22 440-459 91 91 91 151 151 151 F F F 23 460-479 86 86 86 142 142 142 F F F 24 480-499 91 91 91 151 151 151 F F F 25 500-519 91 91 91 151 151 151 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 87 87 87 143 143 143 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 91 91 91 151 151 151 F F F Max gen: 93, Max flush: 155 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.609854443Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.618363451Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.618669084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.618693195Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.618702475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.618714765Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Crucible 586a7b90-55e5-448c-945d-ddce8b4f6ae1 has session id: 865e56ae-f1e3-4060-83e1-41e8cd93d016","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619052799Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Upstairs opts: Upstairs UUID: 586a7b90-55e5-448c-945d-ddce8b4f6ae1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619098329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Crucible stats registered with UUID: 586a7b90-55e5-448c-945d-ddce8b4f6ae1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619111309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:41.619121079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619304592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619320862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619331632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619480974Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619500414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"586a7b90-55e5-448c-945d-ddce8b4f6ae1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619516744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619526214Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619536614Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619546834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619556874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619567565Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619578135Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"ds_connection connected from Ok(127.0.0.1:41554)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619665146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"ds_connection connected from Ok(127.0.0.1:64743)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619691926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"ds_connection connected from Ok(127.0.0.1:40588)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.619703896Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620386494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620400244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620411644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620692457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620716538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[0]R flush_numbers[0..12]: [155, 140, 154, 154, 154, 151, 151, 142, 147, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620728078Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[0]R generation[0..12]: [93, 84, 92, 92, 92, 91, 91, 86, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620739648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620751118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[1]R flush_numbers[0..12]: [155, 140, 154, 154, 154, 151, 151, 142, 147, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620762218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[1]R generation[0..12]: [93, 84, 92, 92, 92, 91, 91, 86, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620773018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620784358Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[2]R flush_numbers[0..12]: [155, 140, 154, 154, 154, 151, 151, 142, 147, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620795359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[2]R generation[0..12]: [93, 84, 92, 92, 92, 91, 91, 86, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620806229Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620817589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620830059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620840219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620850109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620859939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"":"downstairs","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620870349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62087977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"0","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.6208911Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"1","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62090191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"client":"2","":"downstairs","session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.6209123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62092173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"586a7b90-55e5-448c-945d-ddce8b4f6ae1 is now active with session: 865e56ae-f1e3-4060-83e1-41e8cd93d016","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62093219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62094139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"session_id":"865e56ae-f1e3-4060-83e1-41e8cd93d016"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62095093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Activated sub_volume 586a7b90-55e5-448c-945d-ddce8b4f6ae1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.62095944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.620967991Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.621341245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} 01/30 Write block 272 len 24576 data: 7 10 8 10 10 12 02/30 Flush 03/30 Write block 65 len 8192 data: 8 8 04/30 Read block 525 len 4096 05/30 Flush 06/30 Flush 07/30 Write block 502 len 20480 data: 11 9 8 7 7 08/30 Flush 09/30 Write block 81 len 36864 data: 9 10 6 7 8 8 8 7 4 10/30 Write block 544 len 16384 data: 9 10 12 12 11/30 Write block 250 len 28672 data: 9 9 8 9 6 9 12 12/30 Read block 145 len 40960 13/30 Write block 375 len 28672 data: 5 5 4 5 5 5 5 14/30 Write block 459 len 40960 data: 12 12 11 10 9 8 5 5 4 3 15/30 Read block 193 len 4096 16/30 Flush 17/30 Write block 223 len 16384 data: 7 7 9 9 18/30 Read block 499 len 8192 19/30 Write block 404 len 16384 data: 9 9 8 6 20/30 Write block 373 len 24576 data: 3 5 6 6 5 6 21/30 Read block 541 len 28672 22/30 Read block 48 len 4096 23/30 Write block 469 len 12288 data: 4 5 5 24/30 Read block 25 len 16384 25/30 Write block 430 len 24576 data: 11 10 10 11 9 10 26/30 Read block 232 len 16384 27/30 Read block 264 len 32768 28/30 Write block 355 len 20480 data: 6 6 7 8 8 29/30 Write block 486 len 24576 data: 6 5 4 5 7 7 30/30 Write block 347 len 20480 data: 7 6 7 5 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 29 3 35 Done 27 1 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:41.62964045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:41.629939693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1509,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 155 155 155 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 94 92 94 157 154 157 F F F <--- 4 080-099 94 92 94 160 154 160 F F F <--- 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 92 92 92 154 154 154 F F F 11 220-239 92 92 92 154 154 154 T F T <--- 12 240-259 94 93 94 160 155 160 F F F <--- 13 260-279 94 88 94 156 147 156 F T F <--- 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 91 91 91 151 151 151 T F T <--- 18 360-379 94 93 94 160 155 160 T F T <--- 19 380-399 94 93 94 160 155 160 F F F <--- 20 400-419 86 86 86 142 142 142 T F T <--- 21 420-439 93 93 93 155 155 155 T F T <--- 22 440-459 94 91 94 160 151 160 F F F <--- 23 460-479 94 86 94 160 142 160 T F T <--- 24 480-499 91 91 91 151 151 151 T F T <--- 25 500-519 94 91 94 159 151 159 F F F <--- 26 520-539 88 88 88 147 147 147 F F F 27 540-559 94 87 94 160 143 160 F F F <--- 28 560-579 88 88 88 147 147 147 F F F 29 580-599 91 91 91 151 151 151 F F F Max gen: 94, Max flush: 160 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 155 155 155 F F F 1 020-039 84 84 84 140 140 140 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 94 94 94 157 157 157 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 92 92 92 154 154 154 F F F 11 220-239 95 95 95 161 161 161 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 94 94 94 156 156 156 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 90 90 90 150 150 150 F F F 16 320-339 87 87 87 143 143 143 F F F 17 340-359 95 95 95 161 161 161 F F F 18 360-379 95 95 95 161 161 161 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 95 95 95 161 161 161 F F F 22 440-459 94 94 94 160 160 160 F F F 23 460-479 95 95 95 161 161 161 F F F 24 480-499 95 95 95 161 161 161 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 88 88 88 147 147 147 F F F 27 540-559 94 94 94 160 160 160 F F F 28 560-579 88 88 88 147 147 147 F F F 29 580-599 91 91 91 151 151 151 F F F Max gen: 95, Max flush: 161 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.8827617Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891242477Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891535201Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891558761Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891567841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891579921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Crucible 66854a83-c641-4df3-ac7c-0055c81a66de has session id: dd964a65-eb44-4ae5-8288-910667962f7f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891925515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Upstairs opts: Upstairs UUID: 66854a83-c641-4df3-ac7c-0055c81a66de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.891988796Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Crucible stats registered with UUID: 66854a83-c641-4df3-ac7c-0055c81a66de","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892000826Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:45.892012086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892198548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892213849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892224299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.8923481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.89236331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"66854a83-c641-4df3-ac7c-0055c81a66de active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892398171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892408251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892418291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892428321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892438341Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892448841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892459241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51807)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892498192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40239)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892519362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40342)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.892634313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"io task","client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893124249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893141859Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893331311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893471043Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893496013Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[0]R flush_numbers[0..12]: [155, 140, 154, 157, 160, 151, 151, 142, 147, 154, 154, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893679125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[0]R generation[0..12]: [93, 84, 92, 94, 94, 91, 91, 86, 88, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893692075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893703796Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[1]R flush_numbers[0..12]: [155, 140, 154, 157, 160, 151, 151, 142, 147, 154, 154, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893714926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[1]R generation[0..12]: [93, 84, 92, 94, 94, 91, 91, 86, 88, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893725596Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893737066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[2]R flush_numbers[0..12]: [155, 140, 154, 157, 160, 151, 151, 142, 147, 154, 154, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893748036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[2]R generation[0..12]: [93, 84, 92, 94, 94, 91, 91, 86, 88, 92, 92, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893758856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893770096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893779746Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893789807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893799677Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893809677Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"":"downstairs","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893819957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893830807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"0","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893840947Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"1","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893851107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"client":"2","":"downstairs","session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893861227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893870668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"66854a83-c641-4df3-ac7c-0055c81a66de is now active with session: dd964a65-eb44-4ae5-8288-910667962f7f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893881058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893890228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"session_id":"dd964a65-eb44-4ae5-8288-910667962f7f"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893899598Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Activated sub_volume 66854a83-c641-4df3-ac7c-0055c81a66de","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893907928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.893916528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.894313233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} 01/30 Write block 425 len 8192 data: 3 5 02/30 Read block 219 len 4096 03/30 Write block 532 len 28672 data: 5 7 7 7 10 9 12 04/30 Flush 05/30 Write block 277 len 12288 data: 13 14 10 06/30 Read block 100 len 4096 07/30 Read block 321 len 8192 08/30 Write block 488 len 40960 data: 5 6 8 8 6 6 5 4 5 7 09/30 Write block 550 len 32768 data: 10 10 9 8 7 5 7 10 10/30 Read block 377 len 28672 11/30 Flush 12/30 Write block 18 len 36864 data: 6 6 6 6 5 4 4 4 4 13/30 Read block 311 len 20480 14/30 Write block 552 len 28672 data: 10 9 8 6 8 11 13 15/30 Read block 523 len 12288 16/30 Read block 175 len 28672 17/30 Write block 315 len 32768 data: 7 9 9 8 6 6 6 7 18/30 Flush 19/30 Write block 568 len 4096 data: 9 20/30 Write block 592 len 16384 data: 6 6 5 5 21/30 Read block 175 len 16384 22/30 Write block 333 len 4096 data: 10 23/30 Read block 403 len 32768 24/30 Write block 71 len 36864 data: 8 7 6 6 5 5 5 5 6 25/30 Write block 229 len 4096 data: 10 26/30 Write block 474 len 12288 data: 7 6 5 27/30 Write block 583 len 20480 data: 9 12 10 10 9 28/30 Read block 479 len 16384 29/30 Read block 102 len 32768 30/30 Read block 61 len 4096 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 0 29 Done 29 2 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:45.902872591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:45.903125853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1517,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 93 96 164 155 164 F F F <--- 1 020-039 96 84 96 164 140 164 F F F <--- 2 040-059 92 92 92 154 154 154 F F F 3 060-079 94 94 94 157 157 157 T F T <--- 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 92 92 92 154 154 154 F F F 11 220-239 95 95 95 161 161 161 T F T <--- 12 240-259 94 94 94 160 160 160 F F F 13 260-279 96 94 96 163 156 163 F F F <--- 14 280-299 87 87 87 143 143 143 F F F 15 300-319 96 90 96 164 150 164 F F F <--- 16 320-339 96 87 96 164 143 164 T F T <--- 17 340-359 95 95 95 161 161 161 F F F 18 360-379 95 95 95 161 161 161 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 95 96 162 161 162 F T F <--- 22 440-459 94 94 94 160 160 160 F F F 23 460-479 95 95 95 161 161 161 T F T <--- 24 480-499 96 95 96 163 161 163 F F F <--- 25 500-519 94 94 94 159 159 159 F F F 26 520-539 96 88 96 162 147 162 F F F <--- 27 540-559 96 94 96 164 160 164 F F F <--- 28 560-579 88 88 88 147 147 147 T F T <--- 29 580-599 91 91 91 151 151 151 T F T <--- Max gen: 96, Max flush: 164 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 97 97 97 165 165 165 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 92 92 154 154 154 F F F 10 200-219 92 92 92 154 154 154 F F F 11 220-239 97 97 97 165 165 165 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 96 96 96 163 163 163 F F F 14 280-299 87 87 87 143 143 143 F F F 15 300-319 96 96 96 164 164 164 F F F 16 320-339 97 97 97 165 165 165 F F F 17 340-359 95 95 95 161 161 161 F F F 18 360-379 95 95 95 161 161 161 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 160 160 160 F F F 23 460-479 97 97 97 165 165 165 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 96 96 96 162 162 162 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 97 97 97 165 165 165 F F F 29 580-599 97 97 97 165 165 165 F F F Max gen: 97, Max flush: 165 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.151019825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:50.15842336Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.158496251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.165716843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166013497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166037147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166048537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166060467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Crucible 0d35a9a3-e43f-4edb-8ad9-be620ca5dbce has session id: 931c2468-dcee-4ce3-acae-6cc69e488fee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166417631Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Upstairs opts: Upstairs UUID: 0d35a9a3-e43f-4edb-8ad9-be620ca5dbce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166459302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Crucible stats registered with UUID: 0d35a9a3-e43f-4edb-8ad9-be620ca5dbce","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166470722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:50.166480402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166677654Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166710165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166721635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166804686Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166831056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"0d35a9a3-e43f-4edb-8ad9-be620ca5dbce active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166856826Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166866657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166876747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166888727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166898927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166909197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166938457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:39142)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166949877Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:44456)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166960698Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"ds_connection connected from Ok(127.0.0.1:35820)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.166976308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"io task","client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.167634245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.167657096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.167737426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.167952919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.167977939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[0]R flush_numbers[0..12]: [164, 164, 154, 165, 160, 151, 151, 142, 147, 154, 154, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.16802653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[0]R generation[0..12]: [96, 96, 92, 97, 94, 91, 91, 86, 88, 92, 92, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.16803868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.16805038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[1]R flush_numbers[0..12]: [164, 164, 154, 165, 160, 151, 151, 142, 147, 154, 154, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.16806338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[1]R generation[0..12]: [96, 96, 92, 97, 94, 91, 91, 86, 88, 92, 92, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.1680742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.16808639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[2]R flush_numbers[0..12]: [164, 164, 154, 165, 160, 151, 151, 142, 147, 154, 154, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168097291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[2]R generation[0..12]: [96, 96, 92, 97, 94, 91, 91, 86, 88, 92, 92, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168107931Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168119171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168128971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168138911Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168148561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168158261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"":"downstairs","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168168381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168177632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"0","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168187612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"1","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168197642Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"client":"2","":"downstairs","session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168207672Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168218302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"0d35a9a3-e43f-4edb-8ad9-be620ca5dbce is now active with session: 931c2468-dcee-4ce3-acae-6cc69e488fee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168228572Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168237662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"session_id":"931c2468-dcee-4ce3-acae-6cc69e488fee"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168247182Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Activated sub_volume 0d35a9a3-e43f-4edb-8ad9-be620ca5dbce","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168255942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168264423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.168665657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} 01/30 Write block 566 len 16384 data: 9 8 10 5 02/30 Flush 03/30 Flush 04/30 Write block 428 len 32768 data: 6 7 12 11 11 12 10 11 05/30 Read block 0 len 16384 06/30 Read block 421 len 4096 07/30 Read block 121 len 20480 08/30 Write block 199 len 12288 data: 8 7 8 09/30 Read block 122 len 36864 10/30 Read block 25 len 36864 11/30 Write block 354 len 40960 data: 7 7 7 8 9 9 9 8 6 5 12/30 Write block 191 len 40960 data: 12 14 13 13 12 13 10 9 9 8 13/30 Read block 392 len 4096 14/30 Write block 334 len 28672 data: 9 7 7 8 8 9 8 15/30 Flush 16/30 Write block 278 len 36864 data: 15 11 7 5 5 5 6 6 4 17/30 Write block 70 len 16384 data: 8 9 8 7 18/30 Read block 474 len 16384 19/30 Write block 301 len 8192 data: 13 13 20/30 Write block 182 len 40960 data: 9 9 8 11 12 10 8 7 10 13 21/30 Write block 568 len 32768 data: 11 6 5 6 8 9 11 13 22/30 Read block 578 len 16384 23/30 Flush 24/30 Write block 527 len 20480 data: 5 6 5 4 5 25/30 Read block 441 len 40960 26/30 Read block 373 len 24576 27/30 Read block 540 len 28672 28/30 Read block 142 len 16384 29/30 Read block 127 len 40960 30/30 Read block 382 len 24576 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:50.177881693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:50.178156546Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1525,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 97 98 98 165 169 169 F F F <--- 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 92 98 98 154 169 169 F F F <--- 10 200-219 92 98 98 154 168 168 F F F <--- 11 220-239 97 97 97 165 165 165 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 96 98 98 163 169 169 F F F <--- 14 280-299 87 98 98 143 169 169 F F F <--- 15 300-319 96 98 98 164 169 169 F F F <--- 16 320-339 97 98 98 165 168 168 F F F <--- 17 340-359 95 98 98 161 168 168 F F F <--- 18 360-379 95 98 98 161 168 168 F F F <--- 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 98 98 162 168 168 F F F <--- 22 440-459 94 94 94 160 160 160 F F F 23 460-479 97 97 97 165 165 165 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 96 96 96 162 162 162 F T T <--- 27 540-559 96 96 96 164 164 164 F F F 28 560-579 97 98 98 165 169 169 F F F <--- 29 580-599 97 97 97 165 165 165 F F F Max gen: 98, Max flush: 169 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 F F F 2 040-059 92 92 92 154 154 154 F F F 3 060-079 98 98 98 169 169 169 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 91 91 151 151 151 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 98 98 98 169 169 169 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 97 97 97 165 165 165 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 169 169 169 F F F 15 300-319 98 98 98 169 169 169 F F F 16 320-339 98 98 98 168 168 168 F F F 17 340-359 98 98 98 168 168 168 F F F 18 360-379 98 98 98 168 168 168 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 94 94 94 160 160 160 F F F 23 460-479 97 97 97 165 165 165 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 94 94 94 159 159 159 F F F 26 520-539 99 99 99 170 170 170 F F F 27 540-559 96 96 96 164 164 164 F F F 28 560-579 98 98 98 169 169 169 F F F 29 580-599 97 97 97 165 165 165 F F F Max gen: 99, Max flush: 170 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.423506458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:54.431006384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.431082095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438044975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438304208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438327558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438336578Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438348368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Crucible be31fb92-8f58-4a26-a5ae-ed83a652cc44 has session id: 9300e1a7-c585-4d96-8f23-ded131990f3e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438671182Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Upstairs opts: Upstairs UUID: be31fb92-8f58-4a26-a5ae-ed83a652cc44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438700952Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Crucible stats registered with UUID: be31fb92-8f58-4a26-a5ae-ed83a652cc44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438712272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:54.438721902Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438903504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438934945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.438946585Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439059456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439084727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"be31fb92-8f58-4a26-a5ae-ed83a652cc44 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439094657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439104187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439114217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439124217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439134107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439144417Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439154797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44924)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439247228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39833)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439259289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58113)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439270359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"io task","client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439908416Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439930476Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.439942116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.44024646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.44026526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[0]R flush_numbers[0..12]: [164, 164, 154, 169, 160, 151, 151, 142, 147, 169, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440317851Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[0]R generation[0..12]: [96, 96, 92, 98, 94, 91, 91, 86, 88, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440329871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440341631Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[1]R flush_numbers[0..12]: [164, 164, 154, 169, 160, 151, 151, 142, 147, 169, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440352971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[1]R generation[0..12]: [96, 96, 92, 98, 94, 91, 91, 86, 88, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440364031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440375521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[2]R flush_numbers[0..12]: [164, 164, 154, 169, 160, 151, 151, 142, 147, 169, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440389192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[2]R generation[0..12]: [96, 96, 92, 98, 94, 91, 91, 86, 88, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440400152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440411512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440421542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440431992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440442042Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440451912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"":"downstairs","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440462332Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440471632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"0","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440481893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"1","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440491923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"client":"2","":"downstairs","session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440502213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440516223Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"be31fb92-8f58-4a26-a5ae-ed83a652cc44 is now active with session: 9300e1a7-c585-4d96-8f23-ded131990f3e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440526813Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440536283Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"session_id":"9300e1a7-c585-4d96-8f23-ded131990f3e"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440547493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Activated sub_volume be31fb92-8f58-4a26-a5ae-ed83a652cc44","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440556063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440565094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.440865147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} 01/30 Write block 55 len 4096 data: 9 02/30 Read block 529 len 32768 03/30 Read block 205 len 32768 04/30 Read block 411 len 28672 05/30 Read block 157 len 16384 06/30 Write block 579 len 40960 data: 13 12 12 11 10 13 11 11 10 8 07/30 Write block 117 len 12288 data: 5 4 5 08/30 Write block 542 len 8192 data: 8 8 09/30 Read block 128 len 32768 10/30 Flush 11/30 Flush 12/30 Write block 507 len 16384 data: 7 6 7 9 13/30 Read block 511 len 20480 14/30 Read block 283 len 4096 15/30 Write block 416 len 16384 data: 9 7 6 6 16/30 Read block 348 len 4096 17/30 Read block 387 len 36864 18/30 Read block 180 len 24576 19/30 Write block 468 len 32768 data: 4 5 6 6 5 5 8 7 20/30 Write block 332 len 20480 data: 11 11 10 8 8 21/30 Read block 239 len 8192 22/30 Read block 135 len 8192 23/30 Write block 268 len 16384 data: 10 7 6 7 24/30 Read block 145 len 40960 25/30 Write block 197 len 12288 data: 11 10 10 26/30 Read block 206 len 40960 27/30 Write block 508 len 12288 data: 7 8 10 28/30 Read block 257 len 32768 29/30 Read block 382 len 28672 30/30 Read block 551 len 32768 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 2 30 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:54.449594047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:54.44986986Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1533,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 F F F 2 040-059 92 100 100 154 171 171 T F F <--- 3 060-079 98 98 98 169 169 169 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 91 100 100 151 171 171 F F F <--- 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 98 98 98 169 169 169 F T T <--- 10 200-219 98 98 98 168 168 168 F F F 11 220-239 97 97 97 165 165 165 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 98 98 98 169 169 169 F T T <--- 14 280-299 98 98 98 169 169 169 F F F 15 300-319 98 98 98 169 169 169 F F F 16 320-339 98 98 98 168 168 168 F T T <--- 17 340-359 98 98 98 168 168 168 F F F 18 360-379 98 98 98 168 168 168 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 161 161 161 F T T <--- 21 420-439 98 98 98 168 168 168 F F F 22 440-459 94 94 94 160 160 160 F F F 23 460-479 97 97 97 165 165 165 F T T <--- 24 480-499 96 96 96 163 163 163 F F F 25 500-519 94 94 94 159 159 159 F T T <--- 26 520-539 99 99 99 170 170 170 F F F 27 540-559 96 100 100 164 171 171 F F F <--- 28 560-579 98 100 100 169 171 171 F F F <--- 29 580-599 97 100 100 165 171 171 F F F <--- Max gen: 100, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 F F F 2 040-059 100 100 100 171 171 171 F F F 3 060-079 98 98 98 169 169 169 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 97 97 97 165 165 165 F F F 12 240-259 94 94 94 160 160 160 F F F 13 260-279 101 101 101 172 172 172 F F F 14 280-299 98 98 98 169 169 169 F F F 15 300-319 98 98 98 169 169 169 F F F 16 320-339 101 101 101 172 172 172 F F F 17 340-359 98 98 98 168 168 168 F F F 18 360-379 98 98 98 168 168 168 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 94 94 94 160 160 160 F F F 23 460-479 101 101 101 172 172 172 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 101 101 101 172 172 172 F F F 26 520-539 99 99 99 170 170 170 F F F 27 540-559 100 100 100 171 171 171 F F F 28 560-579 100 100 100 171 171 171 F F F 29 580-599 100 100 100 171 171 171 F F F Max gen: 101, Max flush: 172 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.699672854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.708386023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.708683457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.708707007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.708716117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.708727887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Crucible a22701f9-86cd-4c80-b931-f5d2e906de9e has session id: 7c021b59-bf29-4852-bba8-fb9beb58f670","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709119212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Upstairs opts: Upstairs UUID: a22701f9-86cd-4c80-b931-f5d2e906de9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709160222Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Crucible stats registered with UUID: a22701f9-86cd-4c80-b931-f5d2e906de9e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709171122Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:58.709180552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709356194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709387975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709399115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709540266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709566137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"a22701f9-86cd-4c80-b931-f5d2e906de9e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709584237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709593627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709603707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709613697Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709623747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709634278Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709644708Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"ds_connection connected from Ok(127.0.0.1:57179)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709708348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"ds_connection connected from Ok(127.0.0.1:37176)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709735039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"ds_connection connected from Ok(127.0.0.1:38835)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.709777079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"io task","client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710440767Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710466847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710480847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.71075848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710793401Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[0]R flush_numbers[0..12]: [164, 164, 171, 169, 160, 171, 151, 142, 147, 172, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710827341Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[0]R generation[0..12]: [96, 96, 100, 98, 94, 100, 91, 86, 88, 101, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710838821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710850221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[1]R flush_numbers[0..12]: [164, 164, 171, 169, 160, 171, 151, 142, 147, 172, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710861232Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[1]R generation[0..12]: [96, 96, 100, 98, 94, 100, 91, 86, 88, 101, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710872092Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710883462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[2]R flush_numbers[0..12]: [164, 164, 171, 169, 160, 171, 151, 142, 147, 172, 168, 165]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710894252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[2]R generation[0..12]: [96, 96, 100, 98, 94, 100, 91, 86, 88, 101, 98, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710905072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710916312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710926022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710936242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710947473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710957353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"":"downstairs","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710967513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710976683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"0","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710986773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"1","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.710996853Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"client":"2","":"downstairs","session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711007093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711016533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"a22701f9-86cd-4c80-b931-f5d2e906de9e is now active with session: 7c021b59-bf29-4852-bba8-fb9beb58f670","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711026593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711035704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"session_id":"7c021b59-bf29-4852-bba8-fb9beb58f670"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711045174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Activated sub_volume a22701f9-86cd-4c80-b931-f5d2e906de9e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711053494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711062004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.711505509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} 01/30 Write block 590 len 8192 data: 7 6 02/30 Write block 565 len 20480 data: 10 10 9 12 7 03/30 Read block 193 len 16384 04/30 Flush 05/30 Write block 228 len 32768 data: 10 11 9 9 9 8 7 7 06/30 Read block 24 len 12288 07/30 Write block 562 len 32768 data: 10 10 9 11 11 10 13 8 08/30 Read block 456 len 4096 09/30 Write block 324 len 8192 data: 6 6 10/30 Flush 11/30 Write block 309 len 16384 data: 7 6 8 7 12/30 Read block 432 len 40960 13/30 Read block 119 len 40960 14/30 Flush 15/30 Write block 348 len 20480 data: 7 8 6 8 5 16/30 Flush 17/30 Write block 360 len 20480 data: 10 9 7 6 4 18/30 Write block 451 len 12288 data: 7 7 6 19/30 Read block 284 len 36864 20/30 Read block 236 len 20480 21/30 Read block 129 len 40960 22/30 Read block 134 len 4096 23/30 Read block 42 len 4096 24/30 Write block 500 len 28672 data: 12 12 12 10 9 8 8 25/30 Read block 306 len 20480 26/30 Flush 27/30 Write block 246 len 12288 data: 8 8 9 28/30 Write block 50 len 24576 data: 5 7 6 8 7 10 29/30 Read block 243 len 8192 30/30 Write block 25 len 28672 data: 5 5 4 5 5 4 5 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:58.720007676Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:58.72029659Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1541,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 96 96 96 164 164 164 T T F <--- 2 040-059 100 100 100 171 171 171 T T F <--- 3 060-079 98 98 98 169 169 169 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 102 102 97 174 174 165 F F F <--- 12 240-259 94 94 94 160 160 160 T T F <--- 13 260-279 101 101 101 172 172 172 F F F 14 280-299 98 98 98 169 169 169 F F F 15 300-319 102 102 98 175 175 169 F F F <--- 16 320-339 102 102 101 174 174 172 F F F <--- 17 340-359 102 102 98 176 176 168 F F F <--- 18 360-379 102 102 98 177 177 168 F F F <--- 19 380-399 94 94 94 160 160 160 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 102 102 94 177 177 160 F F F <--- 23 460-479 101 101 101 172 172 172 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 102 102 101 177 177 172 F F F <--- 26 520-539 99 99 99 170 170 170 F F F 27 540-559 100 100 100 171 171 171 F F F 28 560-579 102 102 100 174 174 171 F F F <--- 29 580-599 102 102 100 173 173 171 F F F <--- Max gen: 102, Max flush: 177 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 103 103 103 178 178 178 F F F 3 060-079 98 98 98 169 169 169 F F F 4 080-099 94 94 94 160 160 160 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 91 91 91 151 151 151 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 102 102 102 174 174 174 F F F 12 240-259 103 103 103 178 178 178 F F F 13 260-279 101 101 101 172 172 172 F F F 14 280-299 98 98 98 169 169 169 F F F 15 300-319 102 102 102 175 175 175 F F F 16 320-339 102 102 102 174 174 174 F F F 17 340-359 102 102 102 176 176 176 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 102 102 102 177 177 177 F F F 23 460-479 101 101 101 172 172 172 F F F 24 480-499 96 96 96 163 163 163 F F F 25 500-519 102 102 102 177 177 177 F F F 26 520-539 99 99 99 170 170 170 F F F 27 540-559 100 100 100 171 171 171 F F F 28 560-579 102 102 102 174 174 174 F F F 29 580-599 102 102 102 173 173 173 F F F Max gen: 103, Max flush: 178 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.962806689Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.971424817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.971734651Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.971763831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.971774951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.971787221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Crucible 323a303b-b4b8-4ce5-9adf-c95860c016ea has session id: 9520b89c-ab5c-49fc-87b8-c50a4bd0ad56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972155866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Upstairs opts: Upstairs UUID: 323a303b-b4b8-4ce5-9adf-c95860c016ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972195886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Crucible stats registered with UUID: 323a303b-b4b8-4ce5-9adf-c95860c016ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972206786Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:02.972217456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972403649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972419459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972429899Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972590271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972617681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"323a303b-b4b8-4ce5-9adf-c95860c016ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972629271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972638631Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972648861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972660391Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972670292Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972680652Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972691142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"ds_connection connected from Ok(127.0.0.1:57032)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972877574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"ds_connection connected from Ok(127.0.0.1:56248)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.972998015Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"ds_connection connected from Ok(127.0.0.1:44579)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973032836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"io task","client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973554962Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973567082Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973578322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973883655Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973893596Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[0]R flush_numbers[0..12]: [164, 178, 178, 169, 160, 171, 151, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973918656Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[0]R generation[0..12]: [96, 103, 103, 98, 94, 100, 91, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973930366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973941946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[1]R flush_numbers[0..12]: [164, 178, 178, 169, 160, 171, 151, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973954866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[1]R generation[0..12]: [96, 103, 103, 98, 94, 100, 91, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973965766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973976937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[2]R flush_numbers[0..12]: [164, 178, 178, 169, 160, 171, 151, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973987877Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[2]R generation[0..12]: [96, 103, 103, 98, 94, 100, 91, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.973998727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974010047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974020047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974030127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974039737Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974049447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"":"downstairs","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974059857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974069098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"0","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974079538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"1","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974089608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"client":"2","":"downstairs","session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974100908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974110418Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"323a303b-b4b8-4ce5-9adf-c95860c016ea is now active with session: 9520b89c-ab5c-49fc-87b8-c50a4bd0ad56","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974120658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974129868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"session_id":"9520b89c-ab5c-49fc-87b8-c50a4bd0ad56"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974139378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Activated sub_volume 323a303b-b4b8-4ce5-9adf-c95860c016ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974147788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974156359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.974591284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} 01/30 Write block 441 len 4096 data: 10 02/30 Read block 306 len 32768 03/30 Read block 249 len 40960 04/30 Write block 73 len 8192 data: 8 7 05/30 Read block 287 len 20480 06/30 Flush 07/30 Write block 325 len 40960 data: 7 8 9 9 8 8 8 12 12 11 08/30 Read block 110 len 32768 09/30 Flush 10/30 Write block 545 len 20480 data: 11 13 13 8 8 11/30 Read block 185 len 40960 12/30 Write block 595 len 4096 data: 6 13/30 Write block 43 len 16384 data: 8 7 9 7 14/30 Write block 79 len 8192 data: 7 8 15/30 Read block 449 len 12288 16/30 Write block 508 len 24576 data: 8 9 11 10 10 12 17/30 Read block 38 len 16384 18/30 Read block 420 len 32768 19/30 Write block 453 len 36864 data: 7 3 5 7 7 9 13 13 12 20/30 Read block 376 len 24576 21/30 Read block 431 len 8192 22/30 Flush 23/30 Write block 490 len 12288 data: 9 9 7 24/30 Write block 263 len 16384 data: 17 13 15 13 25/30 Write block 319 len 28672 data: 7 7 7 8 8 7 8 26/30 Flush 27/30 Write block 392 len 12288 data: 9 9 7 28/30 Write block 128 len 32768 data: 11 12 11 10 11 12 12 8 29/30 Write block 515 len 4096 data: 9 30/30 Read block 151 len 32768 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:02.98302778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:02.983325234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1549,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 103 181 181 178 F F F <--- 3 060-079 104 104 98 181 181 169 F F F <--- 4 080-099 104 104 94 181 181 160 F F F <--- 5 100-119 100 100 100 171 171 171 F F F 6 120-139 91 91 91 151 151 151 T T F <--- 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 102 102 102 174 174 174 F F F 12 240-259 103 103 103 178 178 178 F F F 13 260-279 104 104 101 182 182 172 F F F <--- 14 280-299 98 98 98 169 169 169 F F F 15 300-319 104 104 102 182 182 175 F F F <--- 16 320-339 104 104 102 182 182 174 F F F <--- 17 340-359 102 102 102 176 176 176 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 94 94 94 160 160 160 T T F <--- 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 102 181 181 177 F F T <--- 23 460-479 104 104 101 181 181 172 F F F <--- 24 480-499 104 104 96 182 182 163 F F F <--- 25 500-519 104 104 102 181 181 177 T T F <--- 26 520-539 99 99 99 170 170 170 F F F 27 540-559 104 104 100 181 181 171 F F F <--- 28 560-579 102 102 102 174 174 174 F F F 29 580-599 104 104 102 181 181 173 F F F <--- Max gen: 104, Max flush: 182 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 105 105 105 183 183 183 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 102 102 102 174 174 174 F F F 12 240-259 103 103 103 178 178 178 F F F 13 260-279 104 104 104 182 182 182 F F F 14 280-299 98 98 98 169 169 169 F F F 15 300-319 104 104 104 182 182 182 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 102 102 102 176 176 176 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 105 105 105 183 183 183 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 99 99 99 170 170 170 F F F 27 540-559 104 104 104 181 181 181 F F F 28 560-579 102 102 102 174 174 174 F F F 29 580-599 104 104 104 181 181 181 F F F Max gen: 105, Max flush: 183 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.229605647Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:07.237142923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.237229314Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.244503357Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24477682Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.244801291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.244810261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.244822441Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Crucible 546b473e-0c95-4587-ab9e-74e2f09c099c has session id: 8b895d5a-32b9-464b-a878-5eb2f1e73da6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245145295Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Upstairs opts: Upstairs UUID: 546b473e-0c95-4587-ab9e-74e2f09c099c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245192815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Crucible stats registered with UUID: 546b473e-0c95-4587-ab9e-74e2f09c099c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245203925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:07.245213405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245391247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245407018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245417538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245534259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245558299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"546b473e-0c95-4587-ab9e-74e2f09c099c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245568739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24557804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24558808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24559803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24560785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24561817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24562861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48761)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245723661Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41332)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245749591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41587)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.245761192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"io task","client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246436019Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.24646184Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246558971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246769953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246780033Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[0]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246894375Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[0]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246906085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246917405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[1]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246928635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[1]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246939505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246950885Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[2]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 147, 172, 168, 174]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246964185Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[2]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 88, 101, 98, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246975066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246986346Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.246996086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247006296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247016196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247025926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"":"downstairs","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247036396Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247045576Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"0","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247056076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"1","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247066367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"client":"2","":"downstairs","session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247076897Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247086497Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"546b473e-0c95-4587-ab9e-74e2f09c099c is now active with session: 8b895d5a-32b9-464b-a878-5eb2f1e73da6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247096627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247105797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"session_id":"8b895d5a-32b9-464b-a878-5eb2f1e73da6"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247116397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Activated sub_volume 546b473e-0c95-4587-ab9e-74e2f09c099c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247124787Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247133247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.247543782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} 01/30 Write block 529 len 40960 data: 6 5 6 6 8 8 8 11 10 13 02/30 Read block 100 len 32768 03/30 Read block 115 len 24576 04/30 Read block 586 len 32768 05/30 Read block 114 len 8192 06/30 Flush 07/30 Write block 543 len 36864 data: 9 10 12 14 14 9 9 11 11 08/30 Read block 126 len 20480 09/30 Write block 221 len 4096 data: 5 10/30 Write block 308 len 40960 data: 6 8 7 9 8 8 7 8 10 10 11/30 Read block 464 len 36864 12/30 Read block 317 len 28672 13/30 Write block 346 len 16384 data: 9 8 8 9 14/30 Write block 351 len 12288 data: 9 6 5 15/30 Flush 16/30 Write block 296 len 24576 data: 10 8 7 9 11 14 17/30 Flush 18/30 Write block 279 len 4096 data: 12 19/30 Read block 17 len 20480 20/30 Read block 14 len 28672 21/30 Read block 55 len 4096 22/30 Read block 299 len 12288 23/30 Write block 248 len 36864 data: 10 9 10 10 9 10 7 10 13 24/30 Flush 25/30 Write block 172 len 12288 data: 7 9 9 26/30 Read block 345 len 12288 27/30 Write block 225 len 12288 data: 10 10 10 28/30 Read block 135 len 24576 29/30 Write block 543 len 8192 data: 10 11 30/30 Write block 233 len 12288 data: 9 8 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 2 32 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:07.256868589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:07.257208793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1557,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 105 105 105 183 183 183 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 88 88 88 147 147 147 F T T <--- 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 102 106 106 174 185 185 F T T <--- 12 240-259 103 106 106 178 187 187 F F F <--- 13 260-279 104 106 106 182 187 187 F F F <--- 14 280-299 98 106 106 169 186 186 F F F <--- 15 300-319 104 106 106 182 186 186 F F F <--- 16 320-339 104 104 104 182 182 182 F F F 17 340-359 102 106 106 176 185 185 F F F <--- 18 360-379 102 102 102 177 177 177 F F F 19 380-399 105 105 105 183 183 183 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 99 106 106 170 184 184 T F F <--- 27 540-559 104 106 106 181 185 185 F T T <--- 28 560-579 102 102 102 174 174 174 F F F 29 580-599 104 104 104 181 181 181 F F F Max gen: 106, Max flush: 187 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 105 105 105 183 183 183 F F F 7 140-159 86 86 86 142 142 142 F F F 8 160-179 107 107 107 188 188 188 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 98 98 168 168 168 F F F 11 220-239 107 107 107 188 188 188 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 187 187 187 F F F 14 280-299 106 106 106 186 186 186 F F F 15 300-319 106 106 106 186 186 186 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 106 106 106 185 185 185 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 105 105 105 183 183 183 F F F 20 400-419 101 101 101 172 172 172 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 106 106 106 184 184 184 F F F 27 540-559 107 107 107 188 188 188 F F F 28 560-579 102 102 102 174 174 174 F F F 29 580-599 104 104 104 181 181 181 F F F Max gen: 107, Max flush: 188 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.499068295Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:11.50647792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.506552241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.513595221Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.513853594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.513877445Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.513886145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.513897905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Crucible 309dfdda-f584-40fd-8d23-5fa5f147d8f9 has session id: c16a3997-9f5c-4a96-a7c4-e2e176b26de8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514218679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Upstairs opts: Upstairs UUID: 309dfdda-f584-40fd-8d23-5fa5f147d8f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514258939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Crucible stats registered with UUID: 309dfdda-f584-40fd-8d23-5fa5f147d8f9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514269609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:11.514281159Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514455271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514485662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514496932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514629973Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514658354Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"309dfdda-f584-40fd-8d23-5fa5f147d8f9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514670744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514680254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514690874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514701354Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514711484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514750285Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514773905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48308)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514835876Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54087)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514860806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62049)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.514891136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"io task","client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515421402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515433843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515614675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515766736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515776696Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[0]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 188, 172, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515916188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[0]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 107, 101, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515928088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515939748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[1]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 188, 172, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515950908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[1]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 107, 101, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515961729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515973139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[2]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 183, 142, 188, 172, 168, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515984409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[2]R generation[0..12]: [96, 103, 104, 104, 104, 100, 105, 86, 107, 101, 98, 107]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.515995369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516006779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516018219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516028439Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516038109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51604779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"":"downstairs","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51605822Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51606737Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"0","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51607731Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"1","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51608722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"client":"2","":"downstairs","session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51609741Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51610687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"309dfdda-f584-40fd-8d23-5fa5f147d8f9 is now active with session: c16a3997-9f5c-4a96-a7c4-e2e176b26de8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51611722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.51612644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"session_id":"c16a3997-9f5c-4a96-a7c4-e2e176b26de8"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516135891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Activated sub_volume 309dfdda-f584-40fd-8d23-5fa5f147d8f9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516144301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516152821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.516532435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} 01/30 Write block 249 len 24576 data: 10 11 11 10 11 8 02/30 Write block 221 len 36864 data: 6 8 8 8 11 11 11 11 12 03/30 Write block 487 len 20480 data: 6 6 7 10 10 04/30 Write block 173 len 4096 data: 10 05/30 Write block 137 len 12288 data: 7 8 7 06/30 Flush 07/30 Write block 395 len 32768 data: 7 7 8 10 7 8 7 7 08/30 Write block 387 len 28672 data: 12 11 10 10 11 10 10 09/30 Read block 390 len 28672 10/30 Write block 218 len 32768 data: 8 6 6 7 9 9 9 12 11/30 Flush 12/30 Write block 576 len 40960 data: 14 11 14 14 13 13 12 11 14 12 13/30 Write block 278 len 28672 data: 16 13 8 6 6 6 7 14/30 Read block 408 len 4096 15/30 Write block 356 len 12288 data: 8 9 10 16/30 Flush 17/30 Write block 317 len 4096 data: 11 18/30 Write block 173 len 28672 data: 11 10 9 10 10 12 12 19/30 Read block 289 len 36864 20/30 Read block 332 len 4096 21/30 Read block 78 len 20480 22/30 Read block 108 len 16384 23/30 Flush 24/30 Write block 414 len 4096 data: 8 25/30 Flush 26/30 Write block 151 len 20480 data: 3 3 4 5 4 27/30 Read block 289 len 40960 28/30 Write block 528 len 40960 data: 7 7 6 7 7 9 9 9 12 11 29/30 Read block 180 len 32768 30/30 Write block 377 len 20480 data: 6 7 6 6 6 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:11.524924011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:11.525226425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1565,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 105 108 108 183 189 189 F F F <--- 7 140-159 86 86 86 142 142 142 F T T <--- 8 160-179 107 108 108 188 192 192 F F F <--- 9 180-199 101 101 101 172 172 172 F F F 10 200-219 98 108 108 168 190 190 F F F <--- 11 220-239 107 108 108 188 190 190 F F F <--- 12 240-259 106 108 108 187 189 189 F F F <--- 13 260-279 106 108 108 187 191 191 F F F <--- 14 280-299 106 108 108 186 191 191 F F F <--- 15 300-319 106 108 108 186 192 192 F F F <--- 16 320-339 104 104 104 182 182 182 F F F 17 340-359 106 108 108 185 191 191 F F F <--- 18 360-379 102 102 102 177 177 177 F T T <--- 19 380-399 105 108 108 183 190 190 F T T <--- 20 400-419 101 108 108 172 193 193 F F F <--- 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 F F F 24 480-499 104 108 108 182 189 189 F F F <--- 25 500-519 105 105 105 183 183 183 F F F 26 520-539 106 106 106 184 184 184 F T T <--- 27 540-559 107 107 107 188 188 188 F F F 28 560-579 102 108 108 174 191 191 F F F <--- 29 580-599 104 108 108 181 191 191 F F F <--- Max gen: 108, Max flush: 193 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 103 103 103 178 178 178 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 109 109 109 194 194 194 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 101 101 101 172 172 172 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 108 108 108 190 190 190 F F F 12 240-259 108 108 108 189 189 189 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 108 108 108 191 191 191 F F F 15 300-319 108 108 108 192 192 192 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 194 194 194 F F F 19 380-399 109 109 109 194 194 194 F F F 20 400-419 108 108 108 193 193 193 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 F F F 24 480-499 108 108 108 189 189 189 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 109 109 109 194 194 194 F F F 27 540-559 107 107 107 188 188 188 F F F 28 560-579 108 108 108 191 191 191 F F F 29 580-599 108 108 108 191 191 191 F F F Max gen: 109, Max flush: 194 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.788045057Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.796858018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797168922Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797200332Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797209862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797221812Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Crucible 7d9be582-253a-458d-8946-a3a106656944 has session id: 7dba8722-c82f-4466-83fa-ddccc4fa0228","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797598966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Upstairs opts: Upstairs UUID: 7d9be582-253a-458d-8946-a3a106656944, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797628467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Crucible stats registered with UUID: 7d9be582-253a-458d-8946-a3a106656944","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797639977Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:15.797652367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797826399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797841559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.797851789Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798009001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798022791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"7d9be582-253a-458d-8946-a3a106656944 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798052622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798062442Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798072512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798082512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798092482Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798124522Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798145833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"ds_connection connected from Ok(127.0.0.1:53755)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798167263Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"ds_connection connected from Ok(127.0.0.1:46072)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798178473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"ds_connection connected from Ok(127.0.0.1:40682)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798189453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"io task","client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798851141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798875151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.798930992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799200035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799222715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[0]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 189, 194, 192, 172, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799299566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[0]R generation[0..12]: [96, 103, 104, 104, 104, 100, 108, 109, 108, 101, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799311646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799323016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[1]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 189, 194, 192, 172, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799334036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[1]R generation[0..12]: [96, 103, 104, 104, 104, 100, 108, 109, 108, 101, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799344976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799356197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[2]R flush_numbers[0..12]: [164, 178, 181, 181, 181, 171, 189, 194, 192, 172, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799367317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[2]R generation[0..12]: [96, 103, 104, 104, 104, 100, 108, 109, 108, 101, 108, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799378157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799389317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799399017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799409097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799418747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799428537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"":"downstairs","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799438888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799449688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"0","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799460098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"1","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799470308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"client":"2","":"downstairs","session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799480798Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799490408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"7d9be582-253a-458d-8946-a3a106656944 is now active with session: 7dba8722-c82f-4466-83fa-ddccc4fa0228","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799500488Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799509678Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"session_id":"7dba8722-c82f-4466-83fa-ddccc4fa0228"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799519238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Activated sub_volume 7d9be582-253a-458d-8946-a3a106656944","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799527729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799536309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.799947883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} 01/30 Write block 30 len 20480 data: 5 6 5 5 4 02/30 Read block 376 len 4096 03/30 Read block 485 len 40960 04/30 Read block 473 len 28672 05/30 Write block 243 len 16384 data: 6 5 7 9 06/30 Read block 265 len 8192 07/30 Read block 547 len 28672 08/30 Read block 484 len 36864 09/30 Read block 187 len 12288 10/30 Read block 550 len 36864 11/30 Read block 569 len 36864 12/30 Read block 579 len 16384 13/30 Read block 255 len 4096 14/30 Write block 290 len 12288 data: 7 6 8 15/30 Flush 16/30 Write block 299 len 16384 data: 10 12 15 14 17/30 Read block 564 len 20480 18/30 Read block 91 len 36864 19/30 Write block 580 len 8192 data: 14 14 20/30 Flush 21/30 Write block 82 len 4096 data: 11 22/30 Read block 559 len 40960 23/30 Write block 321 len 8192 data: 8 9 24/30 Read block 529 len 32768 25/30 Write block 472 len 36864 data: 6 6 9 8 6 5 6 5 4 26/30 Write block 364 len 40960 data: 5 7 6 6 5 8 7 8 5 4 27/30 Read block 167 len 28672 28/30 Read block 141 len 16384 29/30 Write block 199 len 12288 data: 11 9 9 30/30 Read block 71 len 32768 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 1 Done Done Sent false 1002 Acked Read 10 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:15.809159039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:15.809436132Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1573,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 110 110 103 195 195 178 F F T <--- 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 104 104 104 181 181 181 T T F <--- 5 100-119 100 100 100 171 171 171 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 109 109 109 194 194 194 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 101 101 101 172 172 172 T T F <--- 10 200-219 108 108 108 190 190 190 T T F <--- 11 220-239 108 108 108 190 190 190 F F F 12 240-259 110 110 108 195 195 189 F F F <--- 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 108 196 196 191 F F F <--- 15 300-319 110 110 108 196 196 192 F F F <--- 16 320-339 104 104 104 182 182 182 T T F <--- 17 340-359 108 108 108 191 191 191 F F F 18 360-379 109 109 109 194 194 194 T T F <--- 19 380-399 109 109 109 194 194 194 F F F 20 400-419 108 108 108 193 193 193 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 104 104 104 181 181 181 T T F <--- 24 480-499 108 108 108 189 189 189 T T F <--- 25 500-519 105 105 105 183 183 183 F F F 26 520-539 109 109 109 194 194 194 F F F 27 540-559 107 107 107 188 188 188 F F F 28 560-579 108 108 108 191 191 191 F F F 29 580-599 110 110 108 196 196 191 F F F <--- Max gen: 110, Max flush: 196 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 110 110 110 195 195 195 F F F 2 040-059 104 104 104 181 181 181 F F F 3 060-079 104 104 104 181 181 181 F F F 4 080-099 111 111 111 197 197 197 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 109 109 109 194 194 194 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 111 111 111 197 197 197 F F F 10 200-219 111 111 111 197 197 197 F F F 11 220-239 108 108 108 190 190 190 F F F 12 240-259 110 110 110 195 195 195 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 110 110 110 196 196 196 F F F 16 320-339 111 111 111 197 197 197 F F F 17 340-359 108 108 108 191 191 191 F F F 18 360-379 111 111 111 197 197 197 F F F 19 380-399 109 109 109 194 194 194 F F F 20 400-419 108 108 108 193 193 193 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 111 111 111 197 197 197 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 109 109 109 194 194 194 F F F 27 540-559 107 107 107 188 188 188 F F F 28 560-579 108 108 108 191 191 191 F F F 29 580-599 110 110 110 196 196 196 F F F Max gen: 111, Max flush: 197 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.059452047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.067957734Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068258768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068283318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068292608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068304438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Crucible 5c360279-484e-4def-8b10-62fa3cf78376 has session id: a46c4fba-c46e-41a4-9b18-3b7503ac78fc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068688123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Upstairs opts: Upstairs UUID: 5c360279-484e-4def-8b10-62fa3cf78376, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068718283Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Crucible stats registered with UUID: 5c360279-484e-4def-8b10-62fa3cf78376","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068729883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:20.068739653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068929615Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068945526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.068956116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069087377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069111788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"5c360279-484e-4def-8b10-62fa3cf78376 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069126168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069135588Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069145798Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069155938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069165828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069206739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069235879Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41378)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069247759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41919)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069268039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52608)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.06936976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"io task","client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069918687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.069974367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070085419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.07022332Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.07024257Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[0]R flush_numbers[0..12]: [164, 195, 181, 181, 197, 171, 189, 194, 192, 197, 197, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070406802Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[0]R generation[0..12]: [96, 110, 104, 104, 111, 100, 108, 109, 108, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070418692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070430193Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[1]R flush_numbers[0..12]: [164, 195, 181, 181, 197, 171, 189, 194, 192, 197, 197, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070441213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[1]R generation[0..12]: [96, 110, 104, 104, 111, 100, 108, 109, 108, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070452143Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070465273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[2]R flush_numbers[0..12]: [164, 195, 181, 181, 197, 171, 189, 194, 192, 197, 197, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070476213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[2]R generation[0..12]: [96, 110, 104, 104, 111, 100, 108, 109, 108, 111, 111, 108]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070487053Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070498293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070508133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070518334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070528114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070538024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"":"downstairs","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070548164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070557494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"0","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070567634Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"1","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070577764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"client":"2","":"downstairs","session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070588044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070597605Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"5c360279-484e-4def-8b10-62fa3cf78376 is now active with session: a46c4fba-c46e-41a4-9b18-3b7503ac78fc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070608065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070618255Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"session_id":"a46c4fba-c46e-41a4-9b18-3b7503ac78fc"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070627685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Activated sub_volume 5c360279-484e-4def-8b10-62fa3cf78376","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070636175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.070644725Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.07111166Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} 01/30 Write block 576 len 24576 data: 15 12 15 15 15 15 02/30 Write block 217 len 4096 data: 10 03/30 Read block 418 len 8192 04/30 Read block 487 len 36864 05/30 Read block 370 len 4096 06/30 Read block 59 len 40960 07/30 Read block 120 len 28672 08/30 Write block 576 len 24576 data: 16 13 16 16 16 16 09/30 Read block 134 len 40960 10/30 Read block 583 len 24576 11/30 Write block 595 len 4096 data: 7 12/30 Write block 53 len 40960 data: 9 8 11 9 7 8 10 6 6 5 13/30 Write block 154 len 12288 data: 6 5 5 14/30 Write block 574 len 16384 data: 12 14 17 14 15/30 Read block 337 len 28672 16/30 Write block 343 len 12288 data: 11 11 10 17/30 Read block 401 len 16384 18/30 Write block 197 len 28672 data: 12 11 12 10 10 10 9 19/30 Write block 234 len 24576 data: 9 9 8 8 8 9 20/30 Write block 192 len 20480 data: 15 14 14 13 14 21/30 Read block 76 len 24576 22/30 Read block 408 len 8192 23/30 Read block 279 len 16384 24/30 Write block 469 len 36864 data: 6 7 7 7 7 10 9 7 6 25/30 Write block 589 len 8192 data: 7 8 26/30 Write block 380 len 8192 data: 7 7 27/30 Read block 16 len 12288 28/30 Write block 378 len 20480 data: 8 7 8 8 9 29/30 Read block 20 len 24576 30/30 Write block 528 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 6 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 23 26 Done 29 28 7 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:20.079243833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:20.079530787Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1581,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 110 110 110 195 195 195 F F F 2 040-059 104 104 104 181 181 181 T T F <--- 3 060-079 104 104 104 181 181 181 T T F <--- 4 080-099 111 111 111 197 197 197 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 109 109 109 194 194 194 T T F <--- 8 160-179 108 108 108 192 192 192 F F F 9 180-199 111 111 111 197 197 197 T T F <--- 10 200-219 111 111 111 197 197 197 T T T 11 220-239 108 108 108 190 190 190 T T F <--- 12 240-259 110 110 110 195 195 195 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 110 110 110 196 196 196 F F F 16 320-339 111 111 111 197 197 197 F F F 17 340-359 108 108 108 191 191 191 T T F <--- 18 360-379 111 111 111 197 197 197 T T F <--- 19 380-399 109 109 109 194 194 194 T T F <--- 20 400-419 108 108 108 193 193 193 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 111 111 111 197 197 197 T T F <--- 24 480-499 111 111 111 197 197 197 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 109 109 109 194 194 194 T T F <--- 27 540-559 107 107 107 188 188 188 F F F 28 560-579 108 108 108 191 191 191 T T T 29 580-599 110 110 110 196 196 196 T T T Max gen: 111, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 F F F 1 020-039 110 110 110 195 195 195 F F F 2 040-059 112 112 112 198 198 198 F F F 3 060-079 112 112 112 198 198 198 F F F 4 080-099 111 111 111 197 197 197 F F F 5 100-119 100 100 100 171 171 171 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 112 112 112 198 198 198 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 112 112 112 198 198 198 F F F 10 200-219 112 112 112 198 198 198 F F F 11 220-239 112 112 112 198 198 198 F F F 12 240-259 110 110 110 195 195 195 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 110 110 110 196 196 196 F F F 16 320-339 111 111 111 197 197 197 F F F 17 340-359 112 112 112 198 198 198 F F F 18 360-379 112 112 112 198 198 198 F F F 19 380-399 112 112 112 198 198 198 F F F 20 400-419 108 108 108 193 193 193 F F F 21 420-439 98 98 98 168 168 168 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 105 105 105 183 183 183 F F F 26 520-539 112 112 112 198 198 198 F F F 27 540-559 107 107 107 188 188 188 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 112 112 112 198 198 198 F F F Max gen: 112, Max flush: 198 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.332929831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.341457449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.341771303Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.341795953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.341805213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.341817203Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Crucible b3813e63-2eef-46a8-a561-ff0360d323ea has session id: 5f80a476-49c9-41df-8b29-474a72148d6f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342197158Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Upstairs opts: Upstairs UUID: b3813e63-2eef-46a8-a561-ff0360d323ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342230808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Crucible stats registered with UUID: b3813e63-2eef-46a8-a561-ff0360d323ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342242438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:24.342254458Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.34241919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.3424505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342462381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342575722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342607292Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"b3813e63-2eef-46a8-a561-ff0360d323ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342625232Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342634593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342644893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342654963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342669183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342686413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342696693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43133)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342797304Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55257)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342822725Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44752)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.342834405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"io task","client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343421222Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343450372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343538133Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343759455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343795966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[0]R flush_numbers[0..12]: [164, 195, 198, 198, 197, 171, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343856156Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[0]R generation[0..12]: [96, 110, 112, 112, 111, 100, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343867727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343879097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[1]R flush_numbers[0..12]: [164, 195, 198, 198, 197, 171, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343889957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[1]R generation[0..12]: [96, 110, 112, 112, 111, 100, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343900827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343912187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[2]R flush_numbers[0..12]: [164, 195, 198, 198, 197, 171, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343923167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[2]R generation[0..12]: [96, 110, 112, 112, 111, 100, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343933927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343945218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343956228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343966338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343976098Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343985758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"":"downstairs","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.343995948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344005058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"0","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344015238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"1","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344025278Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"client":"2","":"downstairs","session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344035479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344045149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"b3813e63-2eef-46a8-a561-ff0360d323ea is now active with session: 5f80a476-49c9-41df-8b29-474a72148d6f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344055539Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344064679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"session_id":"5f80a476-49c9-41df-8b29-474a72148d6f"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344074279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Activated sub_volume b3813e63-2eef-46a8-a561-ff0360d323ea","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344082799Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344091349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.344470254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} 01/30 Write block 333 len 4096 data: 13 02/30 Read block 454 len 4096 03/30 Write block 98 len 40960 data: 10 10 11 12 13 13 11 14 14 9 04/30 Read block 24 len 24576 05/30 Read block 585 len 4096 06/30 Write block 64 len 8192 data: 5 9 07/30 Read block 241 len 32768 08/30 Read block 439 len 4096 09/30 Read block 421 len 4096 10/30 Read block 358 len 8192 11/30 Read block 177 len 16384 12/30 Read block 383 len 32768 13/30 Read block 193 len 40960 14/30 Read block 372 len 4096 15/30 Write block 418 len 24576 data: 7 7 5 3 2 3 16/30 Write block 44 len 28672 data: 8 10 8 7 6 5 6 17/30 Read block 15 len 28672 18/30 Write block 350 len 24576 data: 7 10 7 6 8 8 19/30 Flush 20/30 Write block 9 len 28672 data: 7 8 9 9 8 10 9 21/30 Read block 428 len 4096 22/30 Read block 357 len 36864 23/30 Write block 519 len 20480 data: 8 9 7 6 5 24/30 Write block 34 len 12288 data: 5 4 4 25/30 Read block 164 len 16384 26/30 Read block 244 len 32768 27/30 Write block 368 len 16384 data: 6 9 8 9 28/30 Write block 555 len 16384 data: 7 9 12 14 29/30 Read block 21 len 12288 30/30 Read block 451 len 16384 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:24.353046472Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:24.353327225Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1589,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 164 164 164 T F T <--- 1 020-039 110 110 110 195 195 195 T F T <--- 2 040-059 114 112 114 199 198 199 F F F <--- 3 060-079 114 112 114 199 198 199 F F F <--- 4 080-099 114 111 114 199 197 199 F F F <--- 5 100-119 114 100 114 199 171 199 F F F <--- 6 120-139 108 108 108 189 189 189 F F F 7 140-159 112 112 112 198 198 198 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 112 112 112 198 198 198 F F F 10 200-219 112 112 112 198 198 198 F F F 11 220-239 112 112 112 198 198 198 F F F 12 240-259 110 110 110 195 195 195 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 110 110 110 196 196 196 F F F 16 320-339 114 111 114 199 197 199 F T F <--- 17 340-359 114 112 114 199 198 199 F F F <--- 18 360-379 112 112 112 198 198 198 T F T <--- 19 380-399 112 112 112 198 198 198 F F F 20 400-419 114 108 114 199 193 199 F F F <--- 21 420-439 114 98 114 199 168 199 F F F <--- 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 105 105 105 183 183 183 T F T <--- 26 520-539 112 112 112 198 198 198 T F T <--- 27 540-559 107 107 107 188 188 188 T F T <--- 28 560-579 112 112 112 198 198 198 F F F 29 580-599 112 112 112 198 198 198 F F F Max gen: 114, Max flush: 199 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 115 115 115 200 200 200 F F F 2 040-059 114 114 114 199 199 199 F F F 3 060-079 114 114 114 199 199 199 F F F 4 080-099 114 114 114 199 199 199 F F F 5 100-119 114 114 114 199 199 199 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 112 112 112 198 198 198 F F F 8 160-179 108 108 108 192 192 192 F F F 9 180-199 112 112 112 198 198 198 F F F 10 200-219 112 112 112 198 198 198 F F F 11 220-239 112 112 112 198 198 198 F F F 12 240-259 110 110 110 195 195 195 F F F 13 260-279 108 108 108 191 191 191 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 110 110 110 196 196 196 F F F 16 320-339 114 114 114 199 199 199 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 115 115 115 200 200 200 F F F 19 380-399 112 112 112 198 198 198 F F F 20 400-419 114 114 114 199 199 199 F F F 21 420-439 114 114 114 199 199 199 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 115 115 115 200 200 200 F F F 27 540-559 115 115 115 200 200 200 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 112 112 112 198 198 198 F F F Max gen: 115, Max flush: 200 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.601143496Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.609631183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.609923996Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.609947476Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.609956467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.609968437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Crucible ae776f3f-cc0e-4854-89ad-a35ed220a17a has session id: 927e88a9-90af-4f42-86f7-b5006a682fc9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610376671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Upstairs opts: Upstairs UUID: ae776f3f-cc0e-4854-89ad-a35ed220a17a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610419082Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Crucible stats registered with UUID: ae776f3f-cc0e-4854-89ad-a35ed220a17a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610430342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:28.610439752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610623204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610654025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610665555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610805566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610824146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"ae776f3f-cc0e-4854-89ad-a35ed220a17a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610840117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610851337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610861517Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610871607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610881607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610892327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.610902817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36798)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.611007999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33238)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.611084319Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35624)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.61110803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"io task","client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.611656426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.611674656Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.611774097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.61196317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.61198058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[0]R flush_numbers[0..12]: [200, 200, 199, 199, 199, 199, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612074751Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[0]R generation[0..12]: [115, 115, 114, 114, 114, 114, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612089031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612100651Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[1]R flush_numbers[0..12]: [200, 200, 199, 199, 199, 199, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612111651Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[1]R generation[0..12]: [115, 115, 114, 114, 114, 114, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612122691Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612134011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[2]R flush_numbers[0..12]: [200, 200, 199, 199, 199, 199, 189, 198, 192, 198, 198, 198]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612145152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[2]R generation[0..12]: [115, 115, 114, 114, 114, 114, 108, 112, 108, 112, 112, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612156072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612167412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612177342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612187602Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612197352Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612207012Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"":"downstairs","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612217172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612226323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"0","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612237483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"1","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612247393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"client":"2","":"downstairs","session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612257683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612267353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"ae776f3f-cc0e-4854-89ad-a35ed220a17a is now active with session: 927e88a9-90af-4f42-86f7-b5006a682fc9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612277613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612286713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"session_id":"927e88a9-90af-4f42-86f7-b5006a682fc9"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612296323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Activated sub_volume ae776f3f-cc0e-4854-89ad-a35ed220a17a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612304683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612313204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.612727048Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} 01/30 Write block 197 len 28672 data: 13 12 13 11 11 11 10 02/30 Read block 137 len 36864 03/30 Read block 485 len 12288 04/30 Write block 278 len 4096 data: 17 05/30 Write block 396 len 12288 data: 8 9 11 06/30 Flush 07/30 Write block 310 len 20480 data: 8 10 9 9 8 08/30 Write block 70 len 32768 data: 9 10 9 9 8 6 6 6 09/30 Write block 240 len 16384 data: 5 7 9 7 10/30 Write block 539 len 36864 data: 10 10 11 9 11 12 13 15 15 11/30 Read block 279 len 12288 12/30 Write block 151 len 12288 data: 4 4 5 13/30 Read block 594 len 24576 14/30 Write block 360 len 28672 data: 11 10 8 7 6 8 7 15/30 Write block 95 len 36864 data: 10 11 13 11 11 12 13 14 14 16/30 Read block 505 len 20480 17/30 Read block 168 len 36864 18/30 Read block 547 len 32768 19/30 Flush 20/30 Write block 390 len 20480 data: 11 12 11 11 8 21/30 Write block 36 len 36864 data: 5 5 4 5 4 5 5 9 9 22/30 Read block 287 len 36864 23/30 Write block 267 len 36864 data: 12 11 8 7 8 8 11 9 11 24/30 Read block 346 len 28672 25/30 Read block 278 len 20480 26/30 Write block 493 len 20480 data: 7 6 5 6 8 27/30 Write block 163 len 8192 data: 10 11 28/30 Write block 227 len 40960 data: 12 12 13 10 10 10 10 10 10 9 29/30 Read block 338 len 12288 30/30 Write block 422 len 28672 data: 3 4 3 4 6 6 7 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:28.621199775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:28.621487199Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1597,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 115 115 115 200 200 200 T F T <--- 2 040-059 114 114 114 199 199 199 T F T <--- 3 060-079 116 114 116 202 199 202 F F F <--- 4 080-099 116 114 116 202 199 202 F F F <--- 5 100-119 116 114 116 202 199 202 F F F <--- 6 120-139 108 108 108 189 189 189 F F F 7 140-159 116 112 116 202 198 202 F F F <--- 8 160-179 108 108 108 192 192 192 T F T <--- 9 180-199 116 112 116 201 198 201 F T F <--- 10 200-219 116 112 116 201 198 201 F T F <--- 11 220-239 112 112 112 198 198 198 T F T <--- 12 240-259 116 110 116 202 195 202 F F F <--- 13 260-279 116 108 116 201 191 201 T F T <--- 14 280-299 110 110 110 196 196 196 F F F 15 300-319 116 110 116 202 196 202 F F F <--- 16 320-339 114 114 114 199 199 199 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 116 115 116 202 200 202 F F F <--- 19 380-399 116 112 116 201 198 201 T F T <--- 20 400-419 114 114 114 199 199 199 F F F 21 420-439 114 114 114 199 199 199 T F T <--- 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 111 111 111 197 197 197 T F T <--- 25 500-519 115 115 115 200 200 200 F F F 26 520-539 116 115 116 202 200 202 F F F <--- 27 540-559 116 115 116 202 200 202 F F F <--- 28 560-579 112 112 112 198 198 198 F F F 29 580-599 112 112 112 198 198 198 F F F Max gen: 116, Max flush: 202 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 117 117 117 203 203 203 F F F 2 040-059 117 117 117 203 203 203 F F F 3 060-079 116 116 116 202 202 202 F F F 4 080-099 116 116 116 202 202 202 F F F 5 100-119 116 116 116 202 202 202 F F F 6 120-139 108 108 108 189 189 189 F F F 7 140-159 116 116 116 202 202 202 F F F 8 160-179 117 117 117 203 203 203 F F F 9 180-199 116 116 116 201 201 201 F F F 10 200-219 116 116 116 201 201 201 F F F 11 220-239 117 117 117 203 203 203 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 117 117 117 203 203 203 F F F 14 280-299 110 110 110 196 196 196 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 114 114 114 199 199 199 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 116 116 116 202 202 202 F F F 19 380-399 117 117 117 203 203 203 F F F 20 400-419 114 114 114 199 199 199 F F F 21 420-439 117 117 117 203 203 203 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 116 116 116 202 202 202 F F F 27 540-559 116 116 116 202 202 202 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 112 112 112 198 198 198 F F F Max gen: 117, Max flush: 203 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.881939803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89044995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.890776174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.890800054Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.890809204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.890823465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Crucible 98e18c96-2495-45f4-9a8b-ad45ba51ef60 has session id: 60829b37-1623-40fe-9c79-1cffb82752b8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891213899Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Upstairs opts: Upstairs UUID: 98e18c96-2495-45f4-9a8b-ad45ba51ef60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891245529Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Crucible stats registered with UUID: 98e18c96-2495-45f4-9a8b-ad45ba51ef60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.8912569Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:32.89126782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891429402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891445452Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891456092Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891609374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891627854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"98e18c96-2495-45f4-9a8b-ad45ba51ef60 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891661504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891671284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891681394Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891691575Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891701675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891713635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891724065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51768)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891798516Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58548)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891870557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49272)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.891893527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"io task","client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892408683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892453233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892595335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892739197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892776747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[0]R flush_numbers[0..12]: [200, 203, 203, 202, 202, 202, 189, 202, 203, 201, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892879798Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[0]R generation[0..12]: [115, 117, 117, 116, 116, 116, 108, 116, 117, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892891688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892903248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[1]R flush_numbers[0..12]: [200, 203, 203, 202, 202, 202, 189, 202, 203, 201, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892914269Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[1]R generation[0..12]: [115, 117, 117, 116, 116, 116, 108, 116, 117, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892925239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892938839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[2]R flush_numbers[0..12]: [200, 203, 203, 202, 202, 202, 189, 202, 203, 201, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892949929Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[2]R generation[0..12]: [115, 117, 117, 116, 116, 116, 108, 116, 117, 116, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892960739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892971849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892981739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.892991889Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89300155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89301151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"":"downstairs","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89302165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89303082Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"0","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89304085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"1","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89305087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"client":"2","":"downstairs","session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89306094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89307038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"98e18c96-2495-45f4-9a8b-ad45ba51ef60 is now active with session: 60829b37-1623-40fe-9c79-1cffb82752b8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.89308085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.893091481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"session_id":"60829b37-1623-40fe-9c79-1cffb82752b8"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.893101121Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Activated sub_volume 98e18c96-2495-45f4-9a8b-ad45ba51ef60","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.893109611Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.893118001Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.893563276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} 01/30 Write block 33 len 36864 data: 6 6 5 6 6 5 6 5 6 02/30 Read block 130 len 12288 03/30 Write block 586 len 24576 data: 12 11 9 8 9 7 04/30 Read block 105 len 40960 05/30 Write block 155 len 32768 data: 6 6 5 5 7 10 9 9 06/30 Read block 46 len 36864 07/30 Flush 08/30 Flush 09/30 Write block 91 len 32768 data: 7 10 9 8 11 12 14 12 10/30 Write block 130 len 24576 data: 12 11 12 13 13 9 11/30 Read block 262 len 4096 12/30 Write block 417 len 36864 data: 8 8 8 6 4 4 5 4 5 13/30 Write block 287 len 36864 data: 5 6 7 8 7 9 8 7 8 14/30 Read block 137 len 36864 15/30 Read block 400 len 32768 16/30 Write block 550 len 12288 data: 12 12 11 17/30 Write block 188 len 20480 data: 9 8 11 14 16 18/30 Write block 78 len 36864 data: 6 8 9 10 12 7 8 9 9 19/30 Flush 20/30 Write block 401 len 36864 data: 8 8 7 10 10 9 7 5 4 21/30 Write block 268 len 8192 data: 12 9 22/30 Read block 227 len 16384 23/30 Read block 585 len 12288 24/30 Write block 67 len 8192 data: 8 7 25/30 Read block 507 len 4096 26/30 Read block 525 len 36864 27/30 Write block 530 len 36864 data: 7 8 8 10 10 10 13 12 14 28/30 Read block 427 len 20480 29/30 Read block 16 len 32768 30/30 Read block 2 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 9 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 25 26 Done 29 30 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:32.902933293Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:32.903218087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1605,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 118 118 117 204 204 203 F F T <--- 2 040-059 118 118 117 204 204 203 F F T <--- 3 060-079 118 118 116 206 206 202 T T F <--- 4 080-099 118 118 116 206 206 202 F F F <--- 5 100-119 116 116 116 202 202 202 F F F 6 120-139 118 118 108 206 206 189 F F F <--- 7 140-159 118 118 116 204 204 202 F F T <--- 8 160-179 118 118 117 204 204 203 F F T <--- 9 180-199 118 118 116 206 206 201 F F F <--- 10 200-219 116 116 116 201 201 201 F F F 11 220-239 117 117 117 203 203 203 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 117 117 117 203 203 203 T T F <--- 14 280-299 118 118 110 206 206 196 F F F <--- 15 300-319 116 116 116 202 202 202 F F F 16 320-339 114 114 114 199 199 199 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 116 116 116 202 202 202 F F F 19 380-399 117 117 117 203 203 203 F F F 20 400-419 118 118 114 206 206 199 T T F <--- 21 420-439 118 118 117 206 206 203 F F F <--- 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 116 116 116 202 202 202 T T F <--- 27 540-559 118 118 116 206 206 202 F F F <--- 28 560-579 112 112 112 198 198 198 F F F 29 580-599 118 118 112 204 204 198 F F T <--- Max gen: 118, Max flush: 206 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 118 118 118 204 204 204 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 119 119 119 207 207 207 F F F 4 080-099 118 118 118 206 206 206 F F F 5 100-119 116 116 116 202 202 202 F F F 6 120-139 118 118 118 206 206 206 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 118 118 118 204 204 204 F F F 9 180-199 118 118 118 206 206 206 F F F 10 200-219 116 116 116 201 201 201 F F F 11 220-239 117 117 117 203 203 203 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 119 119 119 207 207 207 F F F 14 280-299 118 118 118 206 206 206 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 114 114 114 199 199 199 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 116 116 116 202 202 202 F F F 19 380-399 117 117 117 203 203 203 F F F 20 400-419 119 119 119 207 207 207 F F F 21 420-439 118 118 118 206 206 206 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 118 118 118 206 206 206 F F F 28 560-579 112 112 112 198 198 198 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 119, Max flush: 207 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.153339104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.161844551Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162169455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162209545Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162219315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162231665Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Crucible b6bb67e8-6f51-44ab-9915-e543c80c057e has session id: 3dc04e2c-b037-4fd2-b061-b14bc92ac604","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162536959Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Upstairs opts: Upstairs UUID: b6bb67e8-6f51-44ab-9915-e543c80c057e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162573119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Crucible stats registered with UUID: b6bb67e8-6f51-44ab-9915-e543c80c057e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162584759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:37.1625969Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162766732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162796172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162807212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162925273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162951844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"b6bb67e8-6f51-44ab-9915-e543c80c057e active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162978264Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162988074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.162998204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163008444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163018474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163028895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163039155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"ds_connection connected from Ok(127.0.0.1:56693)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163131816Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"ds_connection connected from Ok(127.0.0.1:34665)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163158176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"ds_connection connected from Ok(127.0.0.1:50086)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163169866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"io task","client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163886004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163913025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.163973825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164221048Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164230788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[0]R flush_numbers[0..12]: [200, 204, 204, 207, 206, 202, 206, 204, 204, 206, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164268619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[0]R generation[0..12]: [115, 118, 118, 119, 118, 116, 118, 118, 118, 118, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164280379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164291769Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[1]R flush_numbers[0..12]: [200, 204, 204, 207, 206, 202, 206, 204, 204, 206, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164302749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[1]R generation[0..12]: [115, 118, 118, 119, 118, 116, 118, 118, 118, 118, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164313649Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164324979Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[2]R flush_numbers[0..12]: [200, 204, 204, 207, 206, 202, 206, 204, 204, 206, 201, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16433589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[2]R generation[0..12]: [115, 118, 118, 119, 118, 116, 118, 118, 118, 118, 116, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16434775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16435894Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16437125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16438146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16439113Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16440096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"":"downstairs","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16441091Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.16441993Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"0","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164429821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"1","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164439791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"client":"2","":"downstairs","session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164449961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164459481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"b6bb67e8-6f51-44ab-9915-e543c80c057e is now active with session: 3dc04e2c-b037-4fd2-b061-b14bc92ac604","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164469721Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164478841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"session_id":"3dc04e2c-b037-4fd2-b061-b14bc92ac604"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164488291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Activated sub_volume b6bb67e8-6f51-44ab-9915-e543c80c057e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164496571Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164505071Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.164930386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} 01/30 Flush 02/30 Write block 183 len 28672 data: 10 9 12 13 11 10 9 03/30 Write block 72 len 24576 data: 10 10 9 7 7 7 04/30 Write block 555 len 40960 data: 8 10 13 15 14 11 10 11 11 10 05/30 Write block 266 len 24576 data: 14 13 13 10 8 9 06/30 Read block 171 len 32768 07/30 Write block 274 len 36864 data: 10 12 11 14 18 14 9 7 7 08/30 Flush 09/30 Write block 289 len 40960 data: 8 9 8 10 9 8 9 11 9 8 10/30 Read block 300 len 16384 11/30 Write block 32 len 20480 data: 6 7 7 6 7 12/30 Write block 250 len 20480 data: 12 12 11 12 9 13/30 Read block 525 len 12288 14/30 Read block 342 len 36864 15/30 Write block 447 len 12288 data: 6 7 7 16/30 Read block 382 len 24576 17/30 Flush 18/30 Write block 275 len 32768 data: 13 12 15 19 15 10 8 8 19/30 Write block 241 len 4096 data: 8 20/30 Flush 21/30 Write block 407 len 32768 data: 8 6 5 4 6 7 8 9 22/30 Write block 565 len 12288 data: 12 12 11 23/30 Flush 24/30 Write block 133 len 16384 data: 14 14 10 8 25/30 Write block 339 len 4096 data: 10 26/30 Write block 276 len 20480 data: 13 16 20 16 11 27/30 Read block 159 len 28672 28/30 Read block 505 len 32768 29/30 Write block 211 len 20480 data: 13 12 10 11 10 30/30 Write block 361 len 32768 data: 11 9 8 7 9 8 7 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 7 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:37.173594346Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:37.173878499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1613,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 120 120 118 210 210 204 F F F <--- 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 119 209 209 207 F F F <--- 4 080-099 118 118 118 206 206 206 F F F 5 100-119 116 116 116 202 202 202 F F F 6 120-139 118 118 118 206 206 206 T T F <--- 7 140-159 118 118 118 204 204 204 F F F 8 160-179 118 118 118 204 204 204 F F F 9 180-199 120 120 118 209 209 206 F F F <--- 10 200-219 116 116 116 201 201 201 T T F <--- 11 220-239 117 117 117 203 203 203 F F F 12 240-259 120 120 116 211 211 202 F F F <--- 13 260-279 120 120 119 211 211 207 T T F <--- 14 280-299 120 120 118 211 211 206 T T F <--- 15 300-319 116 116 116 202 202 202 F F F 16 320-339 114 114 114 199 199 199 T T F <--- 17 340-359 114 114 114 199 199 199 F F F 18 360-379 116 116 116 202 202 202 T T F <--- 19 380-399 117 117 117 203 203 203 F F F 20 400-419 120 120 119 212 212 207 F F F <--- 21 420-439 118 118 118 206 206 206 F F F 22 440-459 120 120 104 210 210 181 F F F <--- 23 460-479 112 112 112 198 198 198 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 120 120 118 209 209 206 F F F <--- 28 560-579 120 120 112 212 212 198 F F F <--- 29 580-599 118 118 118 204 204 204 F F F Max gen: 120, Max flush: 212 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 200 200 200 F F F 1 020-039 120 120 120 210 210 210 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 118 118 118 206 206 206 F F F 5 100-119 116 116 116 202 202 202 F F F 6 120-139 121 121 121 213 213 213 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 118 118 118 204 204 204 F F F 9 180-199 120 120 120 209 209 209 F F F 10 200-219 121 121 121 213 213 213 F F F 11 220-239 117 117 117 203 203 203 F F F 12 240-259 120 120 120 211 211 211 F F F 13 260-279 121 121 121 213 213 213 F F F 14 280-299 121 121 121 213 213 213 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 117 117 117 203 203 203 F F F 20 400-419 120 120 120 212 212 212 F F F 21 420-439 118 118 118 206 206 206 F F F 22 440-459 120 120 120 210 210 210 F F F 23 460-479 112 112 112 198 198 198 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 120 120 120 209 209 209 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 118 118 118 204 204 204 F F F Max gen: 121, Max flush: 213 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.426023849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.434540097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43484244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43486838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43487737Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.434889401Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Crucible d95125a2-12a1-4e1c-b3c7-d4610ea93a40 has session id: 45daa960-333b-4542-9c27-47b09d88cb00","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435207694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Upstairs opts: Upstairs UUID: d95125a2-12a1-4e1c-b3c7-d4610ea93a40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435268905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Crucible stats registered with UUID: d95125a2-12a1-4e1c-b3c7-d4610ea93a40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435281305Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:41.435291065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435461477Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435485947Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435496978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435626879Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435648459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"d95125a2-12a1-4e1c-b3c7-d4610ea93a40 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43567552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43568553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43569556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43570561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43571559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43572593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.4357363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:47025)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435795941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:46543)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435807531Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"ds_connection connected from Ok(127.0.0.1:53890)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.435818381Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"io task","client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436426498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436462079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436508699Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436748542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436775202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[0]R flush_numbers[0..12]: [200, 210, 204, 209, 206, 202, 213, 204, 204, 209, 213, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436849423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[0]R generation[0..12]: [115, 120, 118, 120, 118, 116, 121, 118, 118, 120, 121, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436861213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436872723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[1]R flush_numbers[0..12]: [200, 210, 204, 209, 206, 202, 213, 204, 204, 209, 213, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436883833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[1]R generation[0..12]: [115, 120, 118, 120, 118, 116, 121, 118, 118, 120, 121, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436894654Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436905964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[2]R flush_numbers[0..12]: [200, 210, 204, 209, 206, 202, 213, 204, 204, 209, 213, 203]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436916904Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[2]R generation[0..12]: [115, 120, 118, 120, 118, 116, 121, 118, 118, 120, 121, 117]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436927794Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436939314Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436949194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436959364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436969304Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436979204Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"":"downstairs","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.436989465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437000105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"0","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437010265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"1","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437020335Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"client":"2","":"downstairs","session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437030555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437040195Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"d95125a2-12a1-4e1c-b3c7-d4610ea93a40 is now active with session: 45daa960-333b-4542-9c27-47b09d88cb00","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437050515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437059635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"session_id":"45daa960-333b-4542-9c27-47b09d88cb00"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437069336Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Activated sub_volume d95125a2-12a1-4e1c-b3c7-d4610ea93a40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437077746Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.437086226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.43748237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} 01/30 Write block 266 len 32768 data: 15 14 14 11 9 10 9 12 02/30 Write block 593 len 12288 data: 7 6 8 03/30 Write block 118 len 4096 data: 5 04/30 Read block 568 len 16384 05/30 Read block 483 len 36864 06/30 Read block 585 len 12288 07/30 Read block 499 len 20480 08/30 Read block 45 len 28672 09/30 Read block 179 len 40960 10/30 Write block 435 len 20480 data: 12 11 11 11 12 11/30 Flush 12/30 Write block 473 len 4096 data: 8 13/30 Write block 238 len 24576 data: 9 10 6 9 10 8 14/30 Read block 206 len 8192 15/30 Write block 5 len 24576 data: 5 8 8 6 8 9 16/30 Read block 310 len 28672 17/30 Write block 80 len 32768 data: 10 11 13 8 9 10 10 9 18/30 Read block 445 len 28672 19/30 Write block 394 len 20480 data: 9 8 9 10 12 20/30 Read block 252 len 20480 21/30 Read block 543 len 20480 22/30 Flush 23/30 Write block 11 len 16384 data: 10 10 9 11 24/30 Read block 37 len 8192 25/30 Read block 256 len 16384 26/30 Write block 15 len 28672 data: 10 9 8 7 7 7 7 27/30 Read block 541 len 8192 28/30 Write block 185 len 40960 data: 13 14 12 11 10 12 15 17 15 15 29/30 Read block 349 len 12288 30/30 Write block 214 len 32768 data: 12 11 10 11 9 7 7 8 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 20 22 Done 29 29 10 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:41.446549434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:41.446832577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1621,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 115 215 215 200 T T F <--- 1 020-039 120 120 120 210 210 210 T T F <--- 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 118 215 215 206 F F F <--- 5 100-119 122 122 116 214 214 202 F F T <--- 6 120-139 121 121 121 213 213 213 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 118 118 118 204 204 204 F F F 9 180-199 120 120 120 209 209 209 T T F <--- 10 200-219 121 121 121 213 213 213 T T F <--- 11 220-239 122 122 117 215 215 203 T T F <--- 12 240-259 122 122 120 215 215 211 F F F <--- 13 260-279 122 122 121 214 214 213 F F T <--- 14 280-299 121 121 121 213 213 213 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 117 215 215 203 F F F <--- 20 400-419 120 120 120 212 212 212 F F F 21 420-439 122 122 118 214 214 206 F F T <--- 22 440-459 120 120 120 210 210 210 F F F 23 460-479 122 122 112 215 215 198 F F F <--- 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 120 120 120 209 209 209 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 118 214 214 204 F F T <--- Max gen: 122, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 216 216 216 F F F 1 020-039 123 123 123 216 216 216 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 122 122 214 214 214 F F F 6 120-139 121 121 121 213 213 213 F F F 7 140-159 118 118 118 204 204 204 F F F 8 160-179 118 118 118 204 204 204 F F F 9 180-199 123 123 123 216 216 216 F F F 10 200-219 123 123 123 216 216 216 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 122 122 122 215 215 215 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 121 121 121 213 213 213 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F F F 20 400-419 120 120 120 212 212 212 F F F 21 420-439 122 122 122 214 214 214 F F F 22 440-459 120 120 120 210 210 210 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 120 120 120 209 209 209 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 123, Max flush: 216 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.694614978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:45.702149764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.702238245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.709726191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710000474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710046164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710058264Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710070245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Crucible 7d84eba6-1073-473d-9d33-b73723131810 has session id: f6a23f9f-2142-4ab1-9a78-9a2302e9036b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710429149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Upstairs opts: Upstairs UUID: 7d84eba6-1073-473d-9d33-b73723131810, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710477409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Crucible stats registered with UUID: 7d84eba6-1073-473d-9d33-b73723131810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710488289Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:45.710497739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710687042Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710716432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710727562Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710854264Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710872434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"7d84eba6-1073-473d-9d33-b73723131810 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710885744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710894944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710904984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710916374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710926484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710936704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710946775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52629)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.710983945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35060)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.711040606Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53238)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.711109416Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"io task","client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.711720603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.711758944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.711814364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712100768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712125268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[0]R flush_numbers[0..12]: [216, 216, 204, 209, 215, 214, 213, 204, 204, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712211459Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[0]R generation[0..12]: [123, 123, 118, 120, 122, 122, 121, 118, 118, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712223059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712234479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[1]R flush_numbers[0..12]: [216, 216, 204, 209, 215, 214, 213, 204, 204, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712248219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[1]R generation[0..12]: [123, 123, 118, 120, 122, 122, 121, 118, 118, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71225924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71227139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[2]R flush_numbers[0..12]: [216, 216, 204, 209, 215, 214, 213, 204, 204, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71228219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[2]R generation[0..12]: [123, 123, 118, 120, 122, 122, 121, 118, 118, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71229306Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71230409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71231371Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.7123238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.71233431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712343941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"":"downstairs","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712354311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712363431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"0","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712373791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"1","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712383941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"client":"2","":"downstairs","session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712395331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712404861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"7d84eba6-1073-473d-9d33-b73723131810 is now active with session: f6a23f9f-2142-4ab1-9a78-9a2302e9036b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712415061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712424271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"session_id":"f6a23f9f-2142-4ab1-9a78-9a2302e9036b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712433722Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Activated sub_volume 7d84eba6-1073-473d-9d33-b73723131810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712441942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712450362Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.712866987Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} 01/30 Write block 545 len 16384 data: 14 16 16 10 02/30 Read block 199 len 28672 03/30 Write block 402 len 24576 data: 9 8 11 11 10 9 04/30 Write block 154 len 8192 data: 7 7 05/30 Write block 174 len 40960 data: 11 10 11 11 13 13 14 11 10 11 06/30 Write block 400 len 36864 data: 9 9 10 9 12 12 11 10 7 07/30 Read block 306 len 16384 08/30 Read block 246 len 12288 09/30 Read block 579 len 4096 10/30 Write block 15 len 32768 data: 11 10 9 8 8 8 8 6 11/30 Write block 133 len 4096 data: 15 12/30 Write block 208 len 36864 data: 9 10 12 14 13 11 13 12 11 13/30 Read block 26 len 36864 14/30 Read block 47 len 32768 15/30 Write block 20 len 40960 data: 9 9 7 5 5 6 6 5 6 6 16/30 Write block 280 len 40960 data: 12 9 9 7 8 7 5 6 7 9 17/30 Write block 34 len 8192 data: 8 7 18/30 Read block 339 len 8192 19/30 Read block 269 len 16384 20/30 Read block 235 len 16384 21/30 Write block 143 len 20480 data: 5 5 5 5 4 22/30 Read block 338 len 20480 23/30 Read block 316 len 28672 24/30 Write block 540 len 40960 data: 11 12 10 12 13 15 17 17 11 10 25/30 Flush 26/30 Flush 27/30 Write block 455 len 16384 data: 6 8 8 10 28/30 Read block 148 len 32768 29/30 Write block 252 len 24576 data: 12 13 10 11 14 11 30/30 Read block 355 len 4096 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 1 27 Done 4 30 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:45.721848359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:45.722140363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1629,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 124 124 216 217 217 F F F <--- 1 020-039 123 124 124 216 217 217 F F F <--- 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 122 122 214 214 214 F F F 6 120-139 121 124 124 213 217 217 F F F <--- 7 140-159 118 124 124 204 217 217 T F F <--- 8 160-179 118 124 124 204 217 217 F F F <--- 9 180-199 123 124 124 216 217 217 F F F <--- 10 200-219 123 124 124 216 217 217 F F F <--- 11 220-239 123 123 123 216 216 216 F F F 12 240-259 122 122 122 215 215 215 F T T <--- 13 260-279 122 122 122 214 214 214 F F F 14 280-299 121 124 124 213 217 217 F F F <--- 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F F F 20 400-419 120 124 124 212 217 217 T F F <--- 21 420-439 122 122 122 214 214 214 F F F 22 440-459 120 120 120 210 210 210 F T T <--- 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 120 124 124 209 217 217 T F F <--- 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 124, Max flush: 217 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 217 217 217 F F F 1 020-039 124 124 124 217 217 217 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 122 122 214 214 214 F F F 6 120-139 124 124 124 217 217 217 F F F 7 140-159 124 124 124 217 217 217 F F F 8 160-179 124 124 124 217 217 217 F F F 9 180-199 124 124 124 217 217 217 F F F 10 200-219 124 124 124 217 217 217 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 125 125 125 218 218 218 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 124 124 124 217 217 217 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 F F F 17 340-359 114 114 114 199 199 199 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F F F 20 400-419 124 124 124 217 217 217 F F F 21 420-439 122 122 122 214 214 214 F F F 22 440-459 125 125 125 218 218 218 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 F F F 27 540-559 124 124 124 217 217 217 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 125, Max flush: 218 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.974000759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.9828558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983200444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983230114Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983239075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983250995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Crucible 2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b has session id: d982df33-84f4-42b7-92b5-d432e2c0d027","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983572178Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Upstairs opts: Upstairs UUID: 2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983605139Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Crucible stats registered with UUID: 2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983616069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:49.983625599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983800031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983815871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983826261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983970773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.983998343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984014563Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984024014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984034254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984044264Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984054444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984065024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984075394Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"ds_connection connected from Ok(127.0.0.1:43072)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984159945Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"ds_connection connected from Ok(127.0.0.1:37378)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984188555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"ds_connection connected from Ok(127.0.0.1:51380)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984202136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"io task","client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984868273Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984895393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.984907424Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985203627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985214077Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[0]R flush_numbers[0..12]: [217, 217, 204, 209, 215, 214, 217, 217, 217, 217, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985250638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[0]R generation[0..12]: [124, 124, 118, 120, 122, 122, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985262428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985273688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[1]R flush_numbers[0..12]: [217, 217, 204, 209, 215, 214, 217, 217, 217, 217, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985284638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[1]R generation[0..12]: [124, 124, 118, 120, 122, 122, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985295568Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985306628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[2]R flush_numbers[0..12]: [217, 217, 204, 209, 215, 214, 217, 217, 217, 217, 217, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985317608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[2]R generation[0..12]: [124, 124, 118, 120, 122, 122, 124, 124, 124, 124, 124, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985331168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985342499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985352259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985362339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985371919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985381729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"":"downstairs","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985391939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985401119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"0","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985411209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"1","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98542123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"client":"2","":"downstairs","session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98543242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.9854419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b is now active with session: d982df33-84f4-42b7-92b5-d432e2c0d027","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98545251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98546183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"session_id":"d982df33-84f4-42b7-92b5-d432e2c0d027"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98547135Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Activated sub_volume 2ee84ee4-c95f-4c4f-9e82-29db9cb48d1b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98547967Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.98548931Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.985895605Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} 01/30 Write block 352 len 32768 data: 8 7 9 9 9 10 11 10 02/30 Write block 551 len 36864 data: 13 12 10 9 9 11 14 16 15 03/30 Write block 192 len 36864 data: 18 16 16 14 15 14 13 14 12 04/30 Write block 452 len 20480 data: 8 8 4 7 9 05/30 Write block 17 len 20480 data: 10 9 9 10 10 06/30 Read block 314 len 32768 07/30 Read block 574 len 20480 08/30 Read block 593 len 16384 09/30 Read block 462 len 32768 10/30 Read block 227 len 20480 11/30 Read block 580 len 36864 12/30 Read block 436 len 24576 13/30 Write block 123 len 28672 data: 6 6 7 9 10 12 13 14/30 Read block 515 len 36864 15/30 Read block 400 len 12288 16/30 Read block 302 len 4096 17/30 Write block 321 len 40960 data: 9 10 9 8 9 9 10 10 9 9 18/30 Read block 330 len 28672 19/30 Read block 266 len 8192 20/30 Write block 284 len 8192 data: 9 8 21/30 Write block 528 len 32768 data: 9 8 8 9 9 11 11 11 22/30 Read block 288 len 8192 23/30 Write block 403 len 40960 data: 10 13 13 12 11 8 6 5 7 8 24/30 Read block 34 len 12288 25/30 Write block 202 len 8192 data: 12 11 26/30 Read block 300 len 36864 27/30 Read block 174 len 36864 28/30 Read block 88 len 8192 29/30 Write block 0 len 20480 data: 3 6 5 6 5 30/30 Read block 52 len 36864 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 0 28 Done 29 3 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:49.994450843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:49.994732106Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1637,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 217 217 217 T F T <--- 1 020-039 124 124 124 217 217 217 T F T <--- 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 122 122 214 214 214 F F F 6 120-139 124 124 124 217 217 217 T F T <--- 7 140-159 124 124 124 217 217 217 F F F 8 160-179 124 124 124 217 217 217 F F F 9 180-199 124 124 124 217 217 217 T T T 10 200-219 124 124 124 217 217 217 T T T 11 220-239 123 123 123 216 216 216 F F F 12 240-259 125 125 125 218 218 218 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 124 124 124 217 217 217 T F T <--- 15 300-319 116 116 116 202 202 202 F F F 16 320-339 121 121 121 213 213 213 T F T <--- 17 340-359 114 114 114 199 199 199 T T T 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F F F 20 400-419 124 124 124 217 217 217 T F T <--- 21 420-439 122 122 122 214 214 214 F F F 22 440-459 125 125 125 218 218 218 T F T <--- 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 119 119 119 207 207 207 T F T <--- 27 540-559 124 124 124 217 217 217 T T T 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 125, Max flush: 218 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 126 126 126 219 219 219 F F F 2 040-059 118 118 118 204 204 204 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 122 122 214 214 214 F F F 6 120-139 126 126 126 219 219 219 F F F 7 140-159 124 124 124 217 217 217 F F F 8 160-179 124 124 124 217 217 217 F F F 9 180-199 126 126 126 219 219 219 F F F 10 200-219 126 126 126 219 219 219 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 125 125 125 218 218 218 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 126 126 126 219 219 219 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 126 126 126 219 219 219 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 122 122 122 214 214 214 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 115 115 200 200 200 F F F 26 520-539 126 126 126 219 219 219 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 126, Max flush: 219 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.24193173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:54.249388865Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.249462716Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.256720819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.256988142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257023553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257033263Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257045513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Crucible c9072566-cdce-4b40-a93b-b76c9a9837d4 has session id: f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257393077Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Upstairs opts: Upstairs UUID: c9072566-cdce-4b40-a93b-b76c9a9837d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257423487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Crucible stats registered with UUID: c9072566-cdce-4b40-a93b-b76c9a9837d4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257435127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:54.257445197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257616309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.25763261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.25764331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257768891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257791041Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"c9072566-cdce-4b40-a93b-b76c9a9837d4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257807051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257816232Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257826342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257836552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257846712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257857282Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257892992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53289)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257929003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40161)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257940663Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35852)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.257951663Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"io task","client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.258647771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.258669241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.258742542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.258963995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.258989825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[0]R flush_numbers[0..12]: [219, 219, 204, 209, 215, 214, 219, 217, 217, 219, 219, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259048186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[0]R generation[0..12]: [126, 126, 118, 120, 122, 122, 126, 124, 124, 126, 126, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259059976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259071646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[1]R flush_numbers[0..12]: [219, 219, 204, 209, 215, 214, 219, 217, 217, 219, 219, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259082816Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[1]R generation[0..12]: [126, 126, 118, 120, 122, 122, 126, 124, 124, 126, 126, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259093766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259105216Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[2]R flush_numbers[0..12]: [219, 219, 204, 209, 215, 214, 219, 217, 217, 219, 219, 216]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259116266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[2]R generation[0..12]: [126, 126, 118, 120, 122, 122, 126, 124, 124, 126, 126, 123]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259127217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259138567Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259148517Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259160287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259170237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259180017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"":"downstairs","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259190317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259199527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"0","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259209658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"1","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259219718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"client":"2","":"downstairs","session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259230108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259239718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"c9072566-cdce-4b40-a93b-b76c9a9837d4 is now active with session: f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259250118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259259288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"session_id":"f7a11a75-f1b7-4e8b-9006-f67fcd9e18c5"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259269028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Activated sub_volume c9072566-cdce-4b40-a93b-b76c9a9837d4","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259277748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259286298Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.259696823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} 01/30 Write block 174 len 20480 data: 12 11 12 12 14 02/30 Read block 168 len 20480 03/30 Write block 21 len 40960 data: 11 8 6 6 7 7 6 7 7 6 04/30 Read block 222 len 28672 05/30 Read block 87 len 24576 06/30 Read block 562 len 40960 07/30 Read block 412 len 32768 08/30 Write block 230 len 8192 data: 11 11 09/30 Write block 507 len 28672 data: 8 9 10 12 11 11 13 10/30 Write block 100 len 24576 data: 13 14 15 15 12 15 11/30 Read block 496 len 28672 12/30 Read block 154 len 8192 13/30 Read block 354 len 12288 14/30 Read block 121 len 36864 15/30 Read block 86 len 28672 16/30 Write block 147 len 28672 data: 5 3 4 3 5 5 6 17/30 Read block 241 len 16384 18/30 Flush 19/30 Write block 425 len 28672 data: 6 7 7 8 8 13 12 20/30 Read block 361 len 12288 21/30 Write block 381 len 12288 data: 9 10 10 22/30 Write block 247 len 4096 data: 9 23/30 Read block 344 len 28672 24/30 Write block 175 len 24576 data: 12 13 13 15 14 15 25/30 Read block 152 len 32768 26/30 Read block 261 len 4096 27/30 Write block 33 len 36864 data: 8 9 8 8 7 6 7 6 7 28/30 Read block 87 len 16384 29/30 Write block 198 len 28672 data: 14 15 13 12 13 12 9 30/30 Read block 16 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 6 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 4 30 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:54.269699688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:54.269971531Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1645,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 126 128 128 219 220 220 T T T <--- 2 040-059 118 118 118 204 204 204 F T T <--- 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 122 128 128 214 220 220 F F F <--- 6 120-139 126 126 126 219 219 219 F F F 7 140-159 124 128 128 217 220 220 F F F <--- 8 160-179 124 128 128 217 220 220 T T T <--- 9 180-199 126 126 126 219 219 219 F T T <--- 10 200-219 126 126 126 219 219 219 F T T <--- 11 220-239 123 128 128 216 220 220 F F F <--- 12 240-259 125 125 125 218 218 218 F T T <--- 13 260-279 122 122 122 214 214 214 F F F 14 280-299 126 126 126 219 219 219 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 126 126 126 219 219 219 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 122 122 122 215 215 215 F T T <--- 20 400-419 126 126 126 219 219 219 F F F 21 420-439 122 122 122 214 214 214 F T T <--- 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 115 128 128 200 220 220 F F F <--- 26 520-539 126 126 126 219 219 219 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 128, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 129 129 129 221 221 221 F F F 2 040-059 129 129 129 221 221 221 F F F 3 060-079 120 120 120 209 209 209 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 128 128 128 220 220 220 F F F 6 120-139 126 126 126 219 219 219 F F F 7 140-159 128 128 128 220 220 220 F F F 8 160-179 129 129 129 221 221 221 F F F 9 180-199 129 129 129 221 221 221 F F F 10 200-219 129 129 129 221 221 221 F F F 11 220-239 128 128 128 220 220 220 F F F 12 240-259 129 129 129 221 221 221 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 126 126 126 219 219 219 F F F 15 300-319 116 116 116 202 202 202 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 126 126 126 219 219 219 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 129 129 129 221 221 221 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 129 129 129 221 221 221 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 126 126 126 219 219 219 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 129, Max flush: 221 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.521529674Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.530299595Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.530597648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.530620618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.530629888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.530641729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Crucible 70c302b9-86d2-41a9-aa96-f5c9523621ba has session id: b40896fb-4658-4521-a3e8-562a94630875","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531006823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Upstairs opts: Upstairs UUID: 70c302b9-86d2-41a9-aa96-f5c9523621ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531064733Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Crucible stats registered with UUID: 70c302b9-86d2-41a9-aa96-f5c9523621ba","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531076524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:58.531086144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531283666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531313116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531324416Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531428178Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531463118Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"70c302b9-86d2-41a9-aa96-f5c9523621ba active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531472168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531483648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531493998Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531504068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531514069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531524339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531534589Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"ds_connection connected from Ok(127.0.0.1:49449)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531570299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"ds_connection connected from Ok(127.0.0.1:35848)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.53160811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"ds_connection connected from Ok(127.0.0.1:58094)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.531690471Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"io task","client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532286007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532328038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532346388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532611151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532646862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[0]R flush_numbers[0..12]: [219, 221, 221, 209, 215, 220, 219, 220, 221, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532703892Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[0]R generation[0..12]: [126, 129, 129, 120, 122, 128, 126, 128, 129, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532719842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532731503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[1]R flush_numbers[0..12]: [219, 221, 221, 209, 215, 220, 219, 220, 221, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532742533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[1]R generation[0..12]: [126, 129, 129, 120, 122, 128, 126, 128, 129, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532753513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532764833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[2]R flush_numbers[0..12]: [219, 221, 221, 209, 215, 220, 219, 220, 221, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532775743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[2]R generation[0..12]: [126, 129, 129, 120, 122, 128, 126, 128, 129, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532786633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532798033Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532807833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532818164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532827994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532837794Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"":"downstairs","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532847884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532857134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"0","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532868344Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"1","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532878354Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"client":"2","":"downstairs","session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532888494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532898104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"70c302b9-86d2-41a9-aa96-f5c9523621ba is now active with session: b40896fb-4658-4521-a3e8-562a94630875","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532908465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532917625Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"session_id":"b40896fb-4658-4521-a3e8-562a94630875"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532927125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Activated sub_volume 70c302b9-86d2-41a9-aa96-f5c9523621ba","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532935565Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.532944065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.53335701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} 01/30 Write block 73 len 4096 data: 11 02/30 Write block 131 len 32768 data: 12 13 16 15 11 9 8 9 03/30 Write block 533 len 16384 data: 12 12 12 14 04/30 Read block 508 len 4096 05/30 Flush 06/30 Write block 179 len 28672 data: 15 16 12 11 12 10 14 07/30 Read block 234 len 32768 08/30 Write block 353 len 28672 data: 8 10 10 10 11 12 11 09/30 Write block 292 len 36864 data: 11 10 9 10 12 10 9 11 13 10/30 Read block 142 len 16384 11/30 Read block 153 len 28672 12/30 Read block 171 len 4096 13/30 Read block 138 len 16384 14/30 Read block 576 len 32768 15/30 Read block 282 len 8192 16/30 Read block 0 len 20480 17/30 Read block 252 len 36864 18/30 Read block 189 len 28672 19/30 Read block 201 len 32768 20/30 Write block 117 len 8192 data: 6 6 21/30 Write block 230 len 20480 data: 12 12 11 11 11 22/30 Write block 71 len 12288 data: 11 11 12 23/30 Read block 341 len 36864 24/30 Read block 161 len 16384 25/30 Write block 346 len 4096 data: 10 26/30 Write block 528 len 24576 data: 10 9 9 10 10 13 27/30 Read block 401 len 28672 28/30 Read block 13 len 20480 29/30 Read block 561 len 12288 30/30 Read block 182 len 28672 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 0 25 Done 25 0 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:58.541665345Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:58.541966268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1653,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 129 129 129 221 221 221 F F F 2 040-059 129 129 129 221 221 221 F F F 3 060-079 130 130 130 222 222 222 T F T <--- 4 080-099 122 122 122 215 215 215 F F F 5 100-119 128 128 128 220 220 220 T F T <--- 6 120-139 130 130 130 222 222 222 F F F 7 140-159 128 128 128 220 220 220 F F F 8 160-179 129 129 129 221 221 221 T F T <--- 9 180-199 129 129 129 221 221 221 T F T <--- 10 200-219 129 129 129 221 221 221 F F F 11 220-239 128 128 128 220 220 220 T F T <--- 12 240-259 129 129 129 221 221 221 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 126 126 126 219 219 219 T F T <--- 15 300-319 116 116 116 202 202 202 T F T <--- 16 320-339 126 126 126 219 219 219 F F F 17 340-359 126 126 126 219 219 219 T F T <--- 18 360-379 121 121 121 213 213 213 F F F 19 380-399 129 129 129 221 221 221 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 129 129 129 221 221 221 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 130 130 130 222 222 222 T F T <--- 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 130, Max flush: 222 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 129 129 129 221 221 221 F F F 2 040-059 129 129 129 221 221 221 F F F 3 060-079 131 131 131 223 223 223 F F F 4 080-099 122 122 122 215 215 215 F F F 5 100-119 131 131 131 223 223 223 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 128 128 128 220 220 220 F F F 8 160-179 131 131 131 223 223 223 F F F 9 180-199 131 131 131 223 223 223 F F F 10 200-219 129 129 129 221 221 221 F F F 11 220-239 131 131 131 223 223 223 F F F 12 240-259 129 129 129 221 221 221 F F F 13 260-279 122 122 122 214 214 214 F F F 14 280-299 131 131 131 223 223 223 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 131 131 131 223 223 223 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 129 129 129 221 221 221 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 129 129 129 221 221 221 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 117 117 117 203 203 203 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 122 122 122 214 214 214 F F F Max gen: 131, Max flush: 223 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.781593265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790033262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790369386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790396856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790406036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790418066Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Crucible 3153436f-9082-4272-9db3-0f286cf79959 has session id: f989fb43-0ed6-49bd-a888-3c890aedd6cd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790828681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Upstairs opts: Upstairs UUID: 3153436f-9082-4272-9db3-0f286cf79959, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790860911Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Crucible stats registered with UUID: 3153436f-9082-4272-9db3-0f286cf79959","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.790872841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:02.790882762Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791067254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791083354Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791094024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791223505Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791248136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"3153436f-9082-4272-9db3-0f286cf79959 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791257456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791267186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791277886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791288096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791298356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791320277Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791331047Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37185)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791388087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36377)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791437028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47690)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791491729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"io task","client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.791986494Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792042825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792176666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792299908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792337038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[0]R flush_numbers[0..12]: [219, 221, 221, 223, 215, 223, 222, 220, 223, 223, 221, 223]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.79246678Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[0]R generation[0..12]: [126, 129, 129, 131, 122, 131, 130, 128, 131, 131, 129, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.79247928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.79249083Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[1]R flush_numbers[0..12]: [219, 221, 221, 223, 215, 223, 222, 220, 223, 223, 221, 223]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.7925023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[1]R generation[0..12]: [126, 129, 129, 131, 122, 131, 130, 128, 131, 131, 129, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.79251433Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.79252592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[2]R flush_numbers[0..12]: [219, 221, 221, 223, 215, 223, 222, 220, 223, 223, 221, 223]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792537061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[2]R generation[0..12]: [126, 129, 129, 131, 122, 131, 130, 128, 131, 131, 129, 131]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792550181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792561791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792571841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792582271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792592101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792601991Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"":"downstairs","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792612181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792621521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"0","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792631552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"1","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792641532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"client":"2","":"downstairs","session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792651962Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792661542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"3153436f-9082-4272-9db3-0f286cf79959 is now active with session: f989fb43-0ed6-49bd-a888-3c890aedd6cd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792672052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792681522Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"session_id":"f989fb43-0ed6-49bd-a888-3c890aedd6cd"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792691042Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Activated sub_volume 3153436f-9082-4272-9db3-0f286cf79959","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792700802Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.792712643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.793067477Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} 01/30 Write block 79 len 12288 data: 9 11 12 02/30 Read block 404 len 32768 03/30 Read block 524 len 4096 04/30 Read block 73 len 28672 05/30 Write block 65 len 20480 data: 10 9 9 8 6 06/30 Read block 322 len 36864 07/30 Write block 389 len 16384 data: 11 12 13 12 08/30 Write block 236 len 20480 data: 10 9 10 11 7 09/30 Write block 171 len 32768 data: 8 8 12 13 13 14 14 16 10/30 Write block 218 len 20480 data: 10 8 8 9 10 11/30 Write block 415 len 32768 data: 9 10 9 9 9 7 5 5 12/30 Flush 13/30 Write block 580 len 16384 data: 17 17 13 12 14/30 Read block 363 len 12288 15/30 Flush 16/30 Write block 29 len 8192 data: 8 7 17/30 Write block 286 len 32768 data: 6 7 8 10 10 9 12 11 18/30 Read block 230 len 16384 19/30 Write block 296 len 8192 data: 13 11 20/30 Flush 21/30 Write block 155 len 16384 data: 8 7 6 6 22/30 Write block 491 len 8192 data: 11 8 23/30 Read block 288 len 32768 24/30 Read block 145 len 28672 25/30 Flush 26/30 Write block 354 len 8192 data: 11 11 27/30 Write block 108 len 12288 data: 8 11 9 28/30 Write block 212 len 32768 data: 14 12 14 13 12 12 11 9 29/30 Write block 259 len 28672 data: 15 13 13 16 18 14 16 30/30 Read block 168 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 4 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 29 31 Done 30 28 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:02.800878736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:02.80123096Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1661,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 132 132 129 226 226 221 F F F <--- 2 040-059 129 129 129 221 221 221 F F F 3 060-079 132 132 131 224 224 223 F F T <--- 4 080-099 132 132 122 224 224 215 F F T <--- 5 100-119 131 131 131 223 223 223 T T F <--- 6 120-139 130 130 130 222 222 222 F F F 7 140-159 132 132 128 227 227 220 F F F <--- 8 160-179 132 132 131 224 224 223 F F F <--- 9 180-199 131 131 131 223 223 223 F F F 10 200-219 132 132 129 224 224 221 T T F <--- 11 220-239 132 132 131 224 224 223 F F F <--- 12 240-259 132 132 129 224 224 221 T T F <--- 13 260-279 122 122 122 214 214 214 T T F <--- 14 280-299 132 132 131 226 226 223 F F F <--- 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 131 131 131 223 223 223 T T F <--- 18 360-379 121 121 121 213 213 213 F F F 19 380-399 132 132 129 224 224 221 F F F <--- 20 400-419 132 132 126 224 224 219 F F F <--- 21 420-439 132 132 129 224 224 221 F F F <--- 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 117 227 227 203 F F F <--- 25 500-519 128 128 128 220 220 220 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 132 132 122 225 225 214 F F F <--- Max gen: 132, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 219 219 219 F F F 1 020-039 132 132 132 226 226 226 F F F 2 040-059 129 129 129 221 221 221 F F F 3 060-079 132 132 132 224 224 224 F F F 4 080-099 132 132 132 224 224 224 F F F 5 100-119 133 133 133 228 228 228 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 132 132 132 227 227 227 F F F 8 160-179 132 132 132 224 224 224 F F F 9 180-199 131 131 131 223 223 223 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 132 132 132 224 224 224 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 133 133 133 228 228 228 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 133 133 133 228 228 228 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 132 132 132 224 224 224 F F F 20 400-419 132 132 132 224 224 224 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 126 126 126 219 219 219 F F F 28 560-579 120 120 120 212 212 212 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.05605514Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.064604878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.064900061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.064966392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.064992602Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065005883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Crucible 8ae7710e-cc01-42bd-9180-090b31749874 has session id: 994cf8f9-ab61-4584-b6c5-a9a9f796a77a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065312536Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Upstairs opts: Upstairs UUID: 8ae7710e-cc01-42bd-9180-090b31749874, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065362927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Crucible stats registered with UUID: 8ae7710e-cc01-42bd-9180-090b31749874","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065374237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:07.065383837Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065577349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065595609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065606339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065720311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065744391Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"8ae7710e-cc01-42bd-9180-090b31749874 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065759631Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065769031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065779391Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065789621Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065799732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065810312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065820772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48055)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065888343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60032)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065899833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61321)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.065910743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"io task","client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.06655379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.066686712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.066745122Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.066888574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.066965965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[0]R flush_numbers[0..12]: [219, 226, 221, 224, 224, 228, 222, 227, 224, 223, 228, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067046766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[0]R generation[0..12]: [126, 132, 129, 132, 132, 133, 130, 132, 132, 131, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067059176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067070776Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[1]R flush_numbers[0..12]: [219, 226, 221, 224, 224, 228, 222, 227, 224, 223, 228, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067081746Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[1]R generation[0..12]: [126, 132, 129, 132, 132, 133, 130, 132, 132, 131, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067092906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067104267Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[2]R flush_numbers[0..12]: [219, 226, 221, 224, 224, 228, 222, 227, 224, 223, 228, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067115297Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[2]R generation[0..12]: [126, 132, 129, 132, 132, 133, 130, 132, 132, 131, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067126327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067137617Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067147447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067157657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067170247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067180267Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"":"downstairs","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067190398Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067199708Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"0","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067209858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"1","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067219778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"client":"2","":"downstairs","session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067230058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067239648Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"8ae7710e-cc01-42bd-9180-090b31749874 is now active with session: 994cf8f9-ab61-4584-b6c5-a9a9f796a77a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067250028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067259308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"session_id":"994cf8f9-ab61-4584-b6c5-a9a9f796a77a"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067268868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Activated sub_volume 8ae7710e-cc01-42bd-9180-090b31749874","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067277399Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067285929Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.067726274Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} 01/30 Write block 384 len 20480 data: 10 11 11 13 12 02/30 Write block 167 len 24576 data: 12 9 10 8 9 9 03/30 Read block 502 len 8192 04/30 Flush 05/30 Write block 157 len 16384 data: 7 7 8 11 06/30 Write block 198 len 20480 data: 15 16 14 13 14 07/30 Read block 171 len 32768 08/30 Write block 357 len 8192 data: 12 13 09/30 Flush 10/30 Write block 1 len 20480 data: 7 6 7 6 6 11/30 Read block 497 len 16384 12/30 Read block 520 len 32768 13/30 Write block 565 len 28672 data: 13 13 12 14 9 6 7 14/30 Write block 264 len 8192 data: 15 17 15/30 Read block 305 len 24576 16/30 Read block 121 len 12288 17/30 Write block 161 len 16384 data: 10 10 11 12 18/30 Write block 556 len 16384 data: 12 15 17 16 19/30 Read block 409 len 32768 20/30 Read block 556 len 20480 21/30 Read block 138 len 24576 22/30 Read block 527 len 28672 23/30 Read block 411 len 8192 24/30 Flush 25/30 Write block 342 len 32768 data: 10 12 12 11 11 9 9 10 26/30 Read block 318 len 32768 27/30 Read block 465 len 24576 28/30 Read block 19 len 36864 29/30 Write block 58 len 40960 data: 9 11 7 7 6 5 6 11 10 10 30/30 Read block 394 len 32768 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Done Done Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 2 27 Done 26 1 24 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1003)) Some(JobId(1023)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:07.076759257Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:07.07704501Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1669,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 126 134 231 219 231 F F F <--- 1 020-039 132 132 132 226 226 226 F F F 2 040-059 129 129 129 221 221 221 T F T <--- 3 060-079 132 132 132 224 224 224 T F T <--- 4 080-099 132 132 132 224 224 224 F F F 5 100-119 133 133 133 228 228 228 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 134 132 134 230 227 230 F T F <--- 8 160-179 134 134 134 231 229 231 F T F <--- 9 180-199 134 131 134 230 223 230 F F F <--- 10 200-219 134 133 134 230 228 230 F F F <--- 11 220-239 132 132 132 224 224 224 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 134 133 134 231 228 231 F F F <--- 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 134 133 134 230 228 230 T F T <--- 18 360-379 121 121 121 213 213 213 F F F 19 380-399 134 134 134 229 229 229 F F F 20 400-419 132 132 132 224 224 224 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 134 126 134 231 219 231 F F F <--- 28 560-579 134 120 134 231 212 231 F F F <--- 29 580-599 132 132 132 225 225 225 F F F Max gen: 134, Max flush: 231 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 231 231 231 F F F 1 020-039 132 132 132 226 226 226 F F F 2 040-059 135 135 135 232 232 232 F F F 3 060-079 135 135 135 232 232 232 F F F 4 080-099 132 132 132 224 224 224 F F F 5 100-119 133 133 133 228 228 228 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 134 134 134 230 230 230 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 134 134 134 230 230 230 F F F 11 220-239 132 132 132 224 224 224 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 135 135 135 232 232 232 F F F 18 360-379 121 121 121 213 213 213 F F F 19 380-399 134 134 134 229 229 229 F F F 20 400-419 132 132 132 224 224 224 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 128 128 128 220 220 220 F F F 26 520-539 131 131 131 223 223 223 F F F 27 540-559 134 134 134 231 231 231 F F F 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 135, Max flush: 232 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.325370627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:11.332755051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.332830202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340120236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340371279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340394579Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340403619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340415579Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Crucible c053ab85-ab06-403f-8a58-20ddf89c7cdd has session id: b011a582-18dd-4bbd-a4ae-93c274dda13a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340818374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Upstairs opts: Upstairs UUID: c053ab85-ab06-403f-8a58-20ddf89c7cdd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340853184Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Crucible stats registered with UUID: c053ab85-ab06-403f-8a58-20ddf89c7cdd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.340864454Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:11.340877444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341070007Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341087127Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341097707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341260339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341274469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"c053ab85-ab06-403f-8a58-20ddf89c7cdd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341286519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341297219Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341307319Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341317349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.34132743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.3413388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.34135005Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63944)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341434591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55874)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341460921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48713)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.341519342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"io task","client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342053998Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342093398Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.34222442Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342353521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342372172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[0]R flush_numbers[0..12]: [231, 226, 232, 232, 224, 228, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342509893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[0]R generation[0..12]: [134, 132, 135, 135, 132, 133, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342521593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342535393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[1]R flush_numbers[0..12]: [231, 226, 232, 232, 224, 228, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342546534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[1]R generation[0..12]: [134, 132, 135, 135, 132, 133, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342557484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342568834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[2]R flush_numbers[0..12]: [231, 226, 232, 232, 224, 228, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342579874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[2]R generation[0..12]: [134, 132, 135, 135, 132, 133, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342590574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342601764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342611594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342621764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342631474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342641235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"":"downstairs","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342651555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342660785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"0","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342670985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"1","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342682435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"client":"2","":"downstairs","session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342692855Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342702625Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"c053ab85-ab06-403f-8a58-20ddf89c7cdd is now active with session: b011a582-18dd-4bbd-a4ae-93c274dda13a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342713025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342722176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"session_id":"b011a582-18dd-4bbd-a4ae-93c274dda13a"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342731806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Activated sub_volume c053ab85-ab06-403f-8a58-20ddf89c7cdd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342740226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.342748636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.343169441Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} 01/30 Write block 520 len 24576 data: 10 8 7 6 6 6 02/30 Read block 66 len 8192 03/30 Read block 192 len 8192 04/30 Read block 262 len 8192 05/30 Read block 4 len 8192 06/30 Read block 516 len 4096 07/30 Read block 67 len 28672 08/30 Write block 520 len 28672 data: 11 9 8 7 7 7 6 09/30 Read block 274 len 36864 10/30 Write block 28 len 20480 data: 8 9 8 7 7 11/30 Write block 371 len 40960 data: 10 6 5 6 7 7 7 9 8 9 12/30 Write block 369 len 20480 data: 10 9 11 7 6 13/30 Read block 272 len 28672 14/30 Read block 116 len 32768 15/30 Write block 25 len 4096 data: 8 16/30 Read block 512 len 40960 17/30 Write block 539 len 40960 data: 11 12 13 11 13 14 16 18 18 12 18/30 Write block 19 len 20480 data: 10 11 12 9 7 19/30 Read block 118 len 40960 20/30 Flush 21/30 Write block 93 len 8192 data: 10 9 22/30 Read block 48 len 40960 23/30 Write block 411 len 32768 data: 8 9 9 10 10 11 10 10 24/30 Read block 558 len 12288 25/30 Read block 258 len 24576 26/30 Write block 104 len 12288 data: 13 16 15 27/30 Write block 518 len 36864 data: 10 9 12 10 9 8 8 8 7 28/30 Read block 122 len 4096 29/30 Read block 533 len 32768 30/30 Write block 87 len 8192 data: 10 8 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:11.351765679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:11.352073883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1677,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 136 136 231 233 233 F F F <--- 1 020-039 132 136 136 226 233 233 F F F <--- 2 040-059 135 135 135 232 232 232 F F F 3 060-079 135 135 135 232 232 232 F F F 4 080-099 132 132 132 224 224 224 F T T <--- 5 100-119 133 133 133 228 228 228 F T T <--- 6 120-139 130 130 130 222 222 222 F F F 7 140-159 134 134 134 230 230 230 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 134 134 134 230 230 230 F F F 11 220-239 132 132 132 224 224 224 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 135 135 135 232 232 232 F F F 18 360-379 121 136 136 213 233 233 F F F <--- 19 380-399 134 136 136 229 233 233 F F F <--- 20 400-419 132 132 132 224 224 224 F T T <--- 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 128 128 128 220 220 220 F T T <--- 26 520-539 131 136 136 223 233 233 T T T <--- 27 540-559 134 136 136 231 233 233 F F F <--- 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 136, Max flush: 233 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 136 136 136 233 233 233 F F F 2 040-059 135 135 135 232 232 232 F F F 3 060-079 135 135 135 232 232 232 F F F 4 080-099 137 137 137 234 234 234 F F F 5 100-119 137 137 137 234 234 234 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 134 134 134 230 230 230 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 134 134 134 230 230 230 F F F 11 220-239 132 132 132 224 224 224 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 135 135 135 232 232 232 F F F 18 360-379 136 136 136 233 233 233 F F F 19 380-399 136 136 136 233 233 233 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 215 215 215 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 137, Max flush: 234 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.600591411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:15.608062017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.608135428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615113988Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.61536928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615392701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615401761Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615415701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Crucible 14f603bb-50dd-46cc-8531-eb751e69a876 has session id: fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615789655Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Upstairs opts: Upstairs UUID: 14f603bb-50dd-46cc-8531-eb751e69a876, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615819576Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Crucible stats registered with UUID: 14f603bb-50dd-46cc-8531-eb751e69a876","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.615831006Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:15.615840776Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616021538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616036718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616047148Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.61617966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.61619874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"14f603bb-50dd-46cc-8531-eb751e69a876 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.61623268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.61624268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616252781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616262901Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616272871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616315771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616338582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38027)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616350172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34799)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616371312Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64342)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.616382202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"io task","client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617134861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617160981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617172991Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617466965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617477065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[0]R flush_numbers[0..12]: [233, 233, 232, 232, 234, 234, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617515885Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[0]R generation[0..12]: [136, 136, 135, 135, 137, 137, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617527745Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617539345Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[1]R flush_numbers[0..12]: [233, 233, 232, 232, 234, 234, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617550275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[1]R generation[0..12]: [136, 136, 135, 135, 137, 137, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617563216Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617574386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[2]R flush_numbers[0..12]: [233, 233, 232, 232, 234, 234, 222, 230, 231, 230, 230, 224]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617585236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[2]R generation[0..12]: [136, 136, 135, 135, 137, 137, 130, 134, 134, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617596036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617607266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617617236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617627386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617637186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617647037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"":"downstairs","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617657267Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617666527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"0","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617676457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"1","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617686627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"client":"2","":"downstairs","session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617697037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617706687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"14f603bb-50dd-46cc-8531-eb751e69a876 is now active with session: fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617717297Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617727717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"session_id":"fb4a3b1b-a260-4c9c-94f7-a278b2c35d5c"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617737418Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Activated sub_volume 14f603bb-50dd-46cc-8531-eb751e69a876","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617745858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.617754438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.618172493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} 01/30 Write block 363 len 8192 data: 9 8 02/30 Flush 03/30 Write block 366 len 8192 data: 9 8 04/30 Read block 257 len 36864 05/30 Read block 503 len 32768 06/30 Read block 423 len 24576 07/30 Write block 342 len 12288 data: 11 13 13 08/30 Read block 539 len 28672 09/30 Read block 550 len 32768 10/30 Read block 179 len 12288 11/30 Flush 12/30 Write block 212 len 12288 data: 15 13 15 13/30 Read block 172 len 12288 14/30 Read block 32 len 12288 15/30 Read block 346 len 36864 16/30 Read block 51 len 4096 17/30 Write block 55 len 12288 data: 12 10 8 18/30 Write block 234 len 16384 data: 12 11 11 10 19/30 Write block 244 len 16384 data: 6 8 10 10 20/30 Write block 392 len 4096 data: 13 21/30 Read block 289 len 36864 22/30 Read block 451 len 12288 23/30 Read block 23 len 12288 24/30 Read block 240 len 4096 25/30 Write block 458 len 36864 data: 11 14 14 13 11 10 9 6 6 26/30 Read block 403 len 36864 27/30 Read block 450 len 40960 28/30 Read block 65 len 36864 29/30 Write block 155 len 20480 data: 9 8 8 8 9 30/30 Read block 171 len 12288 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 0 28 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:15.627362338Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:15.627638771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1685,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 136 136 136 233 233 233 F F F 2 040-059 135 135 135 232 232 232 F T T <--- 3 060-079 135 135 135 232 232 232 F F F 4 080-099 137 137 137 234 234 234 F F F 5 100-119 137 137 137 234 234 234 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 134 134 134 230 230 230 F T T <--- 8 160-179 134 134 134 231 231 231 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 134 134 134 230 230 230 F T T <--- 11 220-239 132 132 132 224 224 224 F T T <--- 12 240-259 133 133 133 228 228 228 F T T <--- 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 135 138 138 232 236 236 F F F <--- 18 360-379 138 138 138 235 236 236 F F F <--- 19 380-399 136 136 136 233 233 233 F T T <--- 20 400-419 137 137 137 234 234 234 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 126 126 126 219 219 219 F T T <--- 23 460-479 122 122 122 215 215 215 F T T <--- 24 480-499 132 132 132 227 227 227 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 138, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 136 136 136 233 233 233 F F F 2 040-059 139 139 139 237 237 237 F F F 3 060-079 135 135 135 232 232 232 F F F 4 080-099 137 137 137 234 234 234 F F F 5 100-119 137 137 137 234 234 234 F F F 6 120-139 130 130 130 222 222 222 F F F 7 140-159 139 139 139 237 237 237 F F F 8 160-179 134 134 134 231 231 231 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 139 139 139 237 237 237 F F F 11 220-239 139 139 139 237 237 237 F F F 12 240-259 139 139 139 237 237 237 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 131 131 131 223 223 223 F F F 16 320-339 126 126 126 219 219 219 F F F 17 340-359 138 138 138 236 236 236 F F F 18 360-379 138 138 138 236 236 236 F F F 19 380-399 139 139 139 237 237 237 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 132 132 132 224 224 224 F F F 22 440-459 139 139 139 237 237 237 F F F 23 460-479 139 139 139 237 237 237 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 F F F 27 540-559 136 136 136 233 233 233 F F F 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 139, Max flush: 237 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.88059104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.88937757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.889667743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.889690864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.889700154Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.889712274Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Crucible 237ab321-ec5f-41b3-9809-c3b985c18f66 has session id: 0cd6debe-1577-4a69-84e7-b0e4fa2ebc93","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890118519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Upstairs opts: Upstairs UUID: 237ab321-ec5f-41b3-9809-c3b985c18f66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890147369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Crucible stats registered with UUID: 237ab321-ec5f-41b3-9809-c3b985c18f66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890158669Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:19.890168339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890344961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890360281Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890370721Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890520433Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890535593Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"237ab321-ec5f-41b3-9809-c3b985c18f66 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890569444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890579314Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890589334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890599254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890609134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890619534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890630034Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"ds_connection connected from Ok(127.0.0.1:44894)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890699605Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"ds_connection connected from Ok(127.0.0.1:39536)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890725726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"ds_connection connected from Ok(127.0.0.1:40778)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.890737226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"io task","client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891343333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891369693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891499714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891635566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891660396Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[0]R flush_numbers[0..12]: [233, 233, 237, 232, 234, 234, 222, 237, 231, 230, 237, 237]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891781858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[0]R generation[0..12]: [136, 136, 139, 135, 137, 137, 130, 139, 134, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891793598Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891805138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[1]R flush_numbers[0..12]: [233, 233, 237, 232, 234, 234, 222, 237, 231, 230, 237, 237]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891816168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[1]R generation[0..12]: [136, 136, 139, 135, 137, 137, 130, 139, 134, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891827138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891838438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[2]R flush_numbers[0..12]: [233, 233, 237, 232, 234, 234, 222, 237, 231, 230, 237, 237]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891849638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[2]R generation[0..12]: [136, 136, 139, 135, 137, 137, 130, 139, 134, 134, 139, 139]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891860469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891871789Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891881709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891893479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891903239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891912989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"":"downstairs","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891923059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891932259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"0","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.891942379Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"1","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89195228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"client":"2","":"downstairs","session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89196261Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89197212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"237ab321-ec5f-41b3-9809-c3b985c18f66 is now active with session: 0cd6debe-1577-4a69-84e7-b0e4fa2ebc93","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89198239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89199147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"session_id":"0cd6debe-1577-4a69-84e7-b0e4fa2ebc93"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89200101Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Activated sub_volume 237ab321-ec5f-41b3-9809-c3b985c18f66","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89200948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.89201801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.892464925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} 01/30 Write block 343 len 20480 data: 14 14 12 12 10 02/30 Read block 291 len 12288 03/30 Write block 432 len 32768 data: 12 13 11 13 12 12 12 13 04/30 Read block 315 len 32768 05/30 Read block 580 len 32768 06/30 Read block 525 len 40960 07/30 Write block 219 len 36864 data: 10 9 10 11 10 10 13 12 13 08/30 Read block 188 len 24576 09/30 Write block 341 len 24576 data: 11 12 15 15 13 13 10/30 Write block 318 len 32768 data: 9 8 8 10 11 10 9 10 11/30 Read block 456 len 20480 12/30 Write block 38 len 24576 data: 7 8 7 8 6 10 13/30 Write block 78 len 4096 data: 7 14/30 Read block 297 len 20480 15/30 Read block 273 len 36864 16/30 Flush 17/30 Write block 454 len 40960 data: 5 8 10 9 12 15 15 14 12 11 18/30 Read block 474 len 24576 19/30 Read block 31 len 32768 20/30 Write block 540 len 36864 data: 13 14 12 14 15 17 19 19 13 21/30 Write block 328 len 20480 data: 11 10 10 9 13 22/30 Write block 152 len 36864 data: 6 7 8 10 9 9 9 10 12 23/30 Read block 81 len 16384 24/30 Flush 25/30 Write block 491 len 32768 data: 12 9 8 7 6 7 9 11 26/30 Read block 152 len 12288 27/30 Read block 499 len 4096 28/30 Write block 201 len 8192 data: 14 15 29/30 Write block 124 len 16384 data: 7 8 10 11 30/30 Write block 45 len 28672 data: 11 9 8 7 6 7 8 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:19.9015818Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:19.901861903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1693,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 136 238 238 233 F F F <--- 2 040-059 140 140 139 238 238 237 T T F <--- 3 060-079 140 140 135 238 238 232 F F F <--- 4 080-099 137 137 137 234 234 234 F F F 5 100-119 137 137 137 234 234 234 F F F 6 120-139 130 130 130 222 222 222 T T F <--- 7 140-159 140 140 139 239 239 237 F F F <--- 8 160-179 140 140 134 239 239 231 F F F <--- 9 180-199 134 134 134 230 230 230 F F F 10 200-219 140 140 139 238 238 237 T T F <--- 11 220-239 140 140 139 238 238 237 F F F <--- 12 240-259 139 139 139 237 237 237 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 140 140 131 238 238 223 F F F <--- 16 320-339 140 140 126 239 239 219 F F F <--- 17 340-359 140 140 138 238 238 236 F F T <--- 18 360-379 138 138 138 236 236 236 F F F 19 380-399 139 139 139 237 237 237 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 132 238 238 224 F F F <--- 22 440-459 140 140 139 239 239 237 F F F <--- 23 460-479 140 140 139 239 239 237 F F F <--- 24 480-499 132 132 132 227 227 227 T T F <--- 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 F F F 27 540-559 140 140 136 239 239 233 F F F <--- 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 140, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 141 141 141 240 240 240 F F F 3 060-079 140 140 140 238 238 238 F F F 4 080-099 137 137 137 234 234 234 F F F 5 100-119 137 137 137 234 234 234 F F F 6 120-139 141 141 141 240 240 240 F F F 7 140-159 140 140 140 239 239 239 F F F 8 160-179 140 140 140 239 239 239 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 141 141 141 240 240 240 F F F 11 220-239 140 140 140 238 238 238 F F F 12 240-259 139 139 139 237 237 237 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 140 140 140 238 238 238 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 140 140 140 238 238 238 F F F 18 360-379 138 138 138 236 236 236 F F F 19 380-399 139 139 139 237 237 237 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 239 239 239 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 141 141 141 240 240 240 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 F F F 27 540-559 140 140 140 239 239 239 F F F 28 560-579 134 134 134 231 231 231 F F F 29 580-599 132 132 132 225 225 225 F F F Max gen: 141, Max flush: 240 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.153264095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.161722582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162033425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162074126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162084006Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162096146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Crucible fce6830a-9799-42cb-8a53-b843a980887b has session id: 02d98494-c0c5-4520-957d-c83a289d68c6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16244653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Upstairs opts: Upstairs UUID: fce6830a-9799-42cb-8a53-b843a980887b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16247175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Crucible stats registered with UUID: fce6830a-9799-42cb-8a53-b843a980887b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16248254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:24.1624958Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162699813Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162727353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162738313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162861055Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162893735Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"fce6830a-9799-42cb-8a53-b843a980887b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162907935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162917365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162927685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162937866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162947926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162958346Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.162968866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35872)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163044067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48999)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163070467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37976)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163082217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"io task","client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163737235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163795795Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.163855856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164026718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164042728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[0]R flush_numbers[0..12]: [233, 238, 240, 238, 234, 234, 240, 239, 239, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164147209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[0]R generation[0..12]: [136, 140, 141, 140, 137, 137, 141, 140, 140, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.1641616Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16417369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[1]R flush_numbers[0..12]: [233, 238, 240, 238, 234, 234, 240, 239, 239, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16418537Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[1]R generation[0..12]: [136, 140, 141, 140, 137, 137, 141, 140, 140, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16419633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16420751Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[2]R flush_numbers[0..12]: [233, 238, 240, 238, 234, 234, 240, 239, 239, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16421836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[2]R generation[0..12]: [136, 140, 141, 140, 137, 137, 141, 140, 140, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16422912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.16424044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164250211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164260541Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164270371Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164280241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"":"downstairs","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164290411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164299751Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"0","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164311241Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"1","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164321211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"client":"2","":"downstairs","session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164331541Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164341032Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"fce6830a-9799-42cb-8a53-b843a980887b is now active with session: 02d98494-c0c5-4520-957d-c83a289d68c6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164351372Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164360552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"session_id":"02d98494-c0c5-4520-957d-c83a289d68c6"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164370192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Activated sub_volume fce6830a-9799-42cb-8a53-b843a980887b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164378752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164387332Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.164803857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} 01/30 Write block 129 len 4096 data: 14 02/30 Write block 492 len 32768 data: 10 9 8 7 8 10 12 12 03/30 Flush 04/30 Write block 93 len 36864 data: 11 10 12 13 15 13 12 14 15 05/30 Write block 244 len 20480 data: 7 9 11 11 11 06/30 Read block 384 len 8192 07/30 Flush 08/30 Write block 357 len 12288 data: 13 14 12 09/30 Read block 282 len 8192 10/30 Write block 368 len 36864 data: 8 11 10 12 8 7 7 8 8 11/30 Write block 561 len 28672 data: 11 12 12 11 14 14 13 12/30 Write block 241 len 8192 data: 10 11 13/30 Read block 499 len 40960 14/30 Read block 130 len 36864 15/30 Flush 16/30 Write block 247 len 16384 data: 12 12 11 13 17/30 Write block 481 len 16384 data: 4 5 7 7 18/30 Write block 86 len 40960 data: 11 11 9 5 5 8 11 12 11 13 19/30 Read block 284 len 24576 20/30 Read block 132 len 32768 21/30 Write block 165 len 40960 data: 12 12 13 10 11 9 10 10 13 14 22/30 Read block 121 len 8192 23/30 Write block 59 len 20480 data: 12 8 8 7 6 24/30 Write block 111 len 36864 data: 10 9 9 7 6 6 7 7 6 25/30 Write block 524 len 20480 data: 9 9 8 6 11 26/30 Write block 574 len 32768 data: 13 15 18 15 17 17 18 18 27/30 Write block 145 len 32768 data: 6 6 6 4 5 4 6 7 28/30 Write block 303 len 36864 data: 15 15 13 10 8 7 9 9 11 29/30 Read block 510 len 16384 30/30 Read block 377 len 8192 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:24.17291819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:24.173196793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1701,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 141 141 141 240 240 240 T F T <--- 3 060-079 140 140 140 238 238 238 T F T <--- 4 080-099 142 137 142 242 234 242 T F T <--- 5 100-119 142 137 142 242 234 242 T F T <--- 6 120-139 142 141 142 241 240 241 F T F <--- 7 140-159 140 140 140 239 239 239 T F T <--- 8 160-179 140 140 140 239 239 239 T F T <--- 9 180-199 134 134 134 230 230 230 F F F 10 200-219 141 141 141 240 240 240 F F F 11 220-239 140 140 140 238 238 238 F F F 12 240-259 142 139 142 243 237 243 T F T <--- 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 140 140 140 238 238 238 T F T <--- 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 140 142 243 238 243 F F F <--- 18 360-379 142 138 142 243 236 243 F F F <--- 19 380-399 139 139 139 237 237 237 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 239 239 239 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 142 141 142 241 240 241 T T T <--- 25 500-519 137 137 137 234 234 234 F F F 26 520-539 137 137 137 234 234 234 T F T <--- 27 540-559 140 140 140 239 239 239 F F F 28 560-579 142 134 142 243 231 243 T F T <--- 29 580-599 132 132 132 225 225 225 T F T <--- Max gen: 142, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 233 233 233 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 143 143 143 244 244 244 F F F 3 060-079 143 143 143 244 244 244 F F F 4 080-099 143 143 143 244 244 244 F F F 5 100-119 143 143 143 244 244 244 F F F 6 120-139 142 142 142 241 241 241 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 143 143 143 244 244 244 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 141 141 141 240 240 240 F F F 11 220-239 140 140 140 238 238 238 F F F 12 240-259 143 143 143 244 244 244 F F F 13 260-279 134 134 134 231 231 231 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 139 139 139 237 237 237 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 140 140 140 238 238 238 F F F 22 440-459 140 140 140 239 239 239 F F F 23 460-479 140 140 140 239 239 239 F F F 24 480-499 143 143 143 244 244 244 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 143 143 143 244 244 244 F F F 27 540-559 140 140 140 239 239 239 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 143, Max flush: 244 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.428181426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.436702403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.436992507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437020037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437029027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437043577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Crucible 3952c30c-7537-4a7a-a77b-9dd26817635b has session id: 69ceab67-6362-4245-8eea-627f91abdf40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437414961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Upstairs opts: Upstairs UUID: 3952c30c-7537-4a7a-a77b-9dd26817635b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437444062Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Crucible stats registered with UUID: 3952c30c-7537-4a7a-a77b-9dd26817635b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437455562Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:28.437465402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437628834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437658254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437669334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437784826Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437803016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"3952c30c-7537-4a7a-a77b-9dd26817635b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437829086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437838906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437848966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437858967Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437868927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437881327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437891727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"ds_connection connected from Ok(127.0.0.1:40276)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437965248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"ds_connection connected from Ok(127.0.0.1:37571)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437976608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"ds_connection connected from Ok(127.0.0.1:34824)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.437987288Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"io task","client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.438744017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.438765167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.438864378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439093361Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439125111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[0]R flush_numbers[0..12]: [233, 238, 244, 244, 244, 244, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439176142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[0]R generation[0..12]: [136, 140, 143, 143, 143, 143, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439187932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439199252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[1]R flush_numbers[0..12]: [233, 238, 244, 244, 244, 244, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439210202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[1]R generation[0..12]: [136, 140, 143, 143, 143, 143, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439223322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439234662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[2]R flush_numbers[0..12]: [233, 238, 244, 244, 244, 244, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439245642Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[2]R generation[0..12]: [136, 140, 143, 143, 143, 143, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439256403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439267493Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439277313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439287303Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439297003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439306823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"":"downstairs","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439316953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439326193Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"0","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439336323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"1","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439346224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"client":"2","":"downstairs","session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439356254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439365624Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"3952c30c-7537-4a7a-a77b-9dd26817635b is now active with session: 69ceab67-6362-4245-8eea-627f91abdf40","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439375924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439385884Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"session_id":"69ceab67-6362-4245-8eea-627f91abdf40"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439395384Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Activated sub_volume 3952c30c-7537-4a7a-a77b-9dd26817635b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439403744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439412374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.439814069Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} 01/30 Flush 02/30 Write block 382 len 28672 data: 11 11 11 12 12 14 13 03/30 Read block 574 len 40960 04/30 Read block 321 len 36864 05/30 Read block 498 len 20480 06/30 Read block 418 len 16384 07/30 Read block 12 len 16384 08/30 Read block 460 len 28672 09/30 Read block 475 len 28672 10/30 Read block 297 len 16384 11/30 Read block 91 len 28672 12/30 Read block 427 len 28672 13/30 Read block 355 len 40960 14/30 Read block 564 len 28672 15/30 Write block 75 len 36864 data: 8 8 8 8 10 12 13 14 9 16/30 Read block 2 len 40960 17/30 Write block 39 len 40960 data: 9 8 9 7 11 10 12 10 9 8 18/30 Write block 102 len 32768 data: 16 16 14 17 16 10 9 12 19/30 Read block 351 len 20480 20/30 Read block 161 len 24576 21/30 Flush 22/30 Write block 436 len 28672 data: 13 13 13 14 10 11 11 23/30 Write block 6 len 8192 data: 9 9 24/30 Flush 25/30 Write block 257 len 36864 data: 12 17 16 14 14 17 19 16 18 26/30 Read block 78 len 32768 27/30 Read block 119 len 8192 28/30 Write block 472 len 32768 data: 8 9 11 10 8 7 7 6 29/30 Read block 334 len 32768 30/30 Read block 314 len 20480 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 0 28 Done 28 2 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1000)) Some(JobId(1023)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:28.44948503Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:28.449766023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1709,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 136 144 247 233 247 F F F <--- 1 020-039 144 140 144 246 238 246 F F F <--- 2 040-059 144 143 144 246 244 246 F F F <--- 3 060-079 144 143 144 246 244 246 F F F <--- 4 080-099 144 143 144 246 244 246 F F F <--- 5 100-119 144 143 144 246 244 246 F F F <--- 6 120-139 142 142 142 241 241 241 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 143 143 143 244 244 244 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 141 141 141 240 240 240 F F F 11 220-239 140 140 140 238 238 238 F F F 12 240-259 143 143 143 244 244 244 T F T <--- 13 260-279 134 134 134 231 231 231 T F T <--- 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 144 139 144 246 237 246 F T F <--- 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 140 144 247 238 247 F F F <--- 22 440-459 144 140 144 247 239 247 F F F <--- 23 460-479 140 140 140 239 239 239 T F T <--- 24 480-499 143 143 143 244 244 244 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 143 143 143 244 244 244 F F F 27 540-559 140 140 140 239 239 239 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 144, Max flush: 247 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 144 144 144 246 246 246 F F F 2 040-059 144 144 144 246 246 246 F F F 3 060-079 144 144 144 246 246 246 F F F 4 080-099 144 144 144 246 246 246 F F F 5 100-119 144 144 144 246 246 246 F F F 6 120-139 142 142 142 241 241 241 F F F 7 140-159 143 143 143 244 244 244 F F F 8 160-179 143 143 143 244 244 244 F F F 9 180-199 134 134 134 230 230 230 F F F 10 200-219 141 141 141 240 240 240 F F F 11 220-239 140 140 140 238 238 238 F F F 12 240-259 145 145 145 248 248 248 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 F F F 18 360-379 142 142 142 243 243 243 F F F 19 380-399 144 144 144 246 246 246 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 144 144 144 247 247 247 F F F 23 460-479 145 145 145 248 248 248 F F F 24 480-499 143 143 143 244 244 244 F F F 25 500-519 137 137 137 234 234 234 F F F 26 520-539 143 143 143 244 244 244 F F F 27 540-559 140 140 140 239 239 239 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 145, Max flush: 248 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.693231704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70167016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.701957374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.701980434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.701989694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702001734Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Crucible b134733b-761c-4109-80ca-9443839901e7 has session id: e0d8ac52-a261-4d64-b2a9-805f3d1aadba","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702369498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Upstairs opts: Upstairs UUID: b134733b-761c-4109-80ca-9443839901e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702410939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Crucible stats registered with UUID: b134733b-761c-4109-80ca-9443839901e7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702422549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:32.702434249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702603851Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702634141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702645692Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702779183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702802413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"b134733b-761c-4109-80ca-9443839901e7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702816693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702826024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702836504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702846434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702856414Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702866694Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702888854Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"ds_connection connected from Ok(127.0.0.1:63750)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702952905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"ds_connection connected from Ok(127.0.0.1:36325)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702972675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"ds_connection connected from Ok(127.0.0.1:62357)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.702983975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.703578182Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.703605383Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.703755724Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.703895856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.703919196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[0]R flush_numbers[0..12]: [247, 246, 246, 246, 246, 246, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704037367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[0]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704049668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704061218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[1]R flush_numbers[0..12]: [247, 246, 246, 246, 246, 246, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704072138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[1]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704082978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704094388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[2]R flush_numbers[0..12]: [247, 246, 246, 246, 246, 246, 241, 244, 244, 230, 240, 238]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704105318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[2]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 143, 143, 134, 141, 140]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704116148Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704127349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704138509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704148729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704158509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704168269Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"":"downstairs","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704178419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704187549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"0","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704197539Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"1","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704207479Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"client":"2","":"downstairs","session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70421768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70422755Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"b134733b-761c-4109-80ca-9443839901e7 is now active with session: e0d8ac52-a261-4d64-b2a9-805f3d1aadba","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70423774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70424684Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"session_id":"e0d8ac52-a261-4d64-b2a9-805f3d1aadba"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70425657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Activated sub_volume b134733b-761c-4109-80ca-9443839901e7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70426506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.70427355Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.704723115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} 01/30 Flush 02/30 Write block 24 len 4096 data: 7 03/30 Read block 203 len 8192 04/30 Read block 293 len 36864 05/30 Read block 577 len 24576 06/30 Write block 70 len 36864 data: 10 12 12 13 10 9 9 9 9 07/30 Write block 496 len 24576 data: 9 11 13 13 13 13 08/30 Write block 178 len 28672 data: 17 16 17 13 12 13 11 09/30 Flush 10/30 Flush 11/30 Write block 156 len 20480 data: 10 10 10 11 13 12/30 Read block 549 len 16384 13/30 Read block 529 len 32768 14/30 Write block 548 len 12288 data: 14 11 13 15/30 Write block 503 len 12288 data: 11 10 9 16/30 Read block 332 len 8192 17/30 Write block 545 len 40960 data: 18 20 20 15 12 14 14 13 11 10 18/30 Write block 371 len 16384 data: 13 9 8 8 19/30 Write block 148 len 4096 data: 5 20/30 Read block 430 len 12288 21/30 Write block 22 len 8192 data: 10 8 22/30 Read block 349 len 28672 23/30 Write block 71 len 32768 data: 13 13 14 11 10 10 10 10 24/30 Read block 591 len 24576 25/30 Write block 224 len 40960 data: 11 14 13 14 13 14 13 13 12 12 26/30 Flush 27/30 Write block 232 len 16384 data: 13 13 13 12 28/30 Write block 214 len 8192 data: 16 14 29/30 Read block 108 len 36864 30/30 Write block 374 len 20480 data: 9 9 9 8 10 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:32.713257433Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:32.713580687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1717,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 144 146 252 246 252 F F F <--- 2 040-059 144 144 144 246 246 246 F F F 3 060-079 146 144 146 252 246 252 F F F <--- 4 080-099 144 144 144 246 246 246 F F F 5 100-119 144 144 144 246 246 246 F F F 6 120-139 142 142 142 241 241 241 F F F 7 140-159 146 143 146 252 244 252 F F F <--- 8 160-179 146 143 146 252 244 252 F F F <--- 9 180-199 146 134 146 250 230 250 F F F <--- 10 200-219 141 141 141 240 240 240 T F T <--- 11 220-239 146 140 146 252 238 252 T F T <--- 12 240-259 145 145 145 248 248 248 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 F F F 18 360-379 146 142 146 252 243 252 T F T <--- 19 380-399 144 144 144 246 246 246 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 144 144 144 247 247 247 F F F 23 460-479 145 145 145 248 248 248 F F F 24 480-499 146 143 146 250 244 250 F F F <--- 25 500-519 146 137 146 252 234 252 F F F <--- 26 520-539 143 143 143 244 244 244 F F F 27 540-559 146 140 146 252 239 252 F F F <--- 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 146, Max flush: 252 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 144 144 144 246 246 246 F F F 3 060-079 146 146 146 252 252 252 F F F 4 080-099 144 144 144 246 246 246 F F F 5 100-119 144 144 144 246 246 246 F F F 6 120-139 142 142 142 241 241 241 F F F 7 140-159 146 146 146 252 252 252 F F F 8 160-179 146 146 146 252 252 252 F F F 9 180-199 146 146 146 250 250 250 F F F 10 200-219 147 147 147 253 253 253 F F F 11 220-239 147 147 147 253 253 253 F F F 12 240-259 145 145 145 248 248 248 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 144 144 144 246 246 246 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 144 144 144 247 247 247 F F F 23 460-479 145 145 145 248 248 248 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 146 146 146 252 252 252 F F F 26 520-539 143 143 143 244 244 244 F F F 27 540-559 146 146 146 252 252 252 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 147, Max flush: 253 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.955694431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96430792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.964629944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.964655464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.964664174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.964676234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Crucible 0b24d80e-8ba9-4da5-a379-1c543ec50044 has session id: f112a4ea-45e3-47d7-bff9-be8011a0f418","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965035848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Upstairs opts: Upstairs UUID: 0b24d80e-8ba9-4da5-a379-1c543ec50044, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965070999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Crucible stats registered with UUID: 0b24d80e-8ba9-4da5-a379-1c543ec50044","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965082349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:36.965096339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965280681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965297861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965308511Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965446093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965473673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"0b24d80e-8ba9-4da5-a379-1c543ec50044 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965489173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965498703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965508844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965518994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965529144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965539764Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965587234Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"ds_connection connected from Ok(127.0.0.1:42286)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965624235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"ds_connection connected from Ok(127.0.0.1:62783)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965636385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"ds_connection connected from Ok(127.0.0.1:38323)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.965647775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966331583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966388484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966424714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966643017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966666377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[0]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 246, 241, 252, 252, 250, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966723507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[0]R generation[0..12]: [144, 146, 144, 146, 144, 144, 142, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966737778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966749438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[1]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 246, 241, 252, 252, 250, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966760708Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[1]R generation[0..12]: [144, 146, 144, 146, 144, 144, 142, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966771558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966783028Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[2]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 246, 241, 252, 252, 250, 253, 253]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966793968Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[2]R generation[0..12]: [144, 146, 144, 146, 144, 144, 142, 146, 146, 146, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966805048Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966816549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966826259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966836579Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966846279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966856179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"":"downstairs","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966866419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966875619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"0","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966887129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"1","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.966897109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"client":"2","":"downstairs","session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96690726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96691699Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"0b24d80e-8ba9-4da5-a379-1c543ec50044 is now active with session: f112a4ea-45e3-47d7-bff9-be8011a0f418","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96693739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96694651Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"session_id":"f112a4ea-45e3-47d7-bff9-be8011a0f418"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96695601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Activated sub_volume 0b24d80e-8ba9-4da5-a379-1c543ec50044","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96696413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.96697272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.967299094Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} 01/30 Write block 244 len 36864 data: 8 10 12 13 13 12 14 13 13 02/30 Read block 342 len 24576 03/30 Write block 347 len 12288 data: 11 10 11 04/30 Write block 171 len 8192 data: 11 11 05/30 Read block 534 len 8192 06/30 Read block 373 len 40960 07/30 Write block 126 len 8192 data: 11 12 08/30 Write block 383 len 28672 data: 12 12 13 13 15 14 12 09/30 Read block 587 len 32768 10/30 Read block 154 len 16384 11/30 Write block 102 len 40960 data: 17 17 15 18 17 11 10 13 10 11 12/30 Read block 552 len 40960 13/30 Write block 225 len 32768 data: 15 14 15 14 15 14 14 14 14/30 Read block 15 len 40960 15/30 Read block 83 len 24576 16/30 Write block 137 len 16384 data: 9 10 8 5 17/30 Write block 550 len 36864 data: 15 15 14 12 11 10 13 16 18 18/30 Read block 374 len 28672 19/30 Read block 288 len 32768 20/30 Read block 168 len 16384 21/30 Read block 220 len 4096 22/30 Read block 353 len 8192 23/30 Read block 538 len 28672 24/30 Write block 548 len 40960 data: 16 13 16 16 15 13 12 11 14 17 25/30 Read block 30 len 4096 26/30 Write block 507 len 32768 data: 9 10 11 13 12 12 14 11 27/30 Read block 116 len 4096 28/30 Read block 11 len 32768 29/30 Write block 216 len 16384 data: 13 13 12 11 30/30 Write block 535 len 24576 data: 13 15 13 15 12 14 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:36.975809011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:36.976087955Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1725,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 144 144 144 246 246 246 F F F 3 060-079 146 146 146 252 252 252 F F F 4 080-099 144 144 144 246 246 246 F F F 5 100-119 144 144 144 246 246 246 T F T <--- 6 120-139 142 142 142 241 241 241 T F T <--- 7 140-159 146 146 146 252 252 252 T F T <--- 8 160-179 146 146 146 252 252 252 T F T <--- 9 180-199 146 146 146 250 250 250 F F F 10 200-219 147 147 147 253 253 253 T F T <--- 11 220-239 147 147 147 253 253 253 T F T <--- 12 240-259 145 145 145 248 248 248 T T T 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 142 142 142 243 243 243 T F T <--- 18 360-379 147 147 147 253 253 253 F F F 19 380-399 144 144 144 246 246 246 T F T <--- 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 144 144 144 247 247 247 F F F 23 460-479 145 145 145 248 248 248 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 146 146 146 252 252 252 T F T <--- 26 520-539 143 143 143 244 244 244 T F T <--- 27 540-559 146 146 146 252 252 252 T F T <--- 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 147, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 146 146 252 252 252 F F F 2 040-059 144 144 144 246 246 246 F F F 3 060-079 146 146 146 252 252 252 F F F 4 080-099 144 144 144 246 246 246 F F F 5 100-119 148 148 148 254 254 254 F F F 6 120-139 148 148 148 254 254 254 F F F 7 140-159 148 148 148 254 254 254 F F F 8 160-179 148 148 148 254 254 254 F F F 9 180-199 146 146 146 250 250 250 F F F 10 200-219 148 148 148 254 254 254 F F F 11 220-239 148 148 148 254 254 254 F F F 12 240-259 148 148 148 254 254 254 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 143 143 244 244 244 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 148 148 148 254 254 254 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F F F 22 440-459 144 144 144 247 247 247 F F F 23 460-479 145 145 145 248 248 248 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 148 148 148 254 254 254 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 148 148 148 254 254 254 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 148, Max flush: 254 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.225067729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:41.232522714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.232597155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.239877368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240135031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240159011Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240170572Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240182542Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Crucible e2a71550-7d60-475f-9c8e-761ee2985739 has session id: a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240483145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Upstairs opts: Upstairs UUID: e2a71550-7d60-475f-9c8e-761ee2985739, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240514736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Crucible stats registered with UUID: e2a71550-7d60-475f-9c8e-761ee2985739","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240526636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:41.240536386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240701508Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240718138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240728708Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240850449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24087645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"e2a71550-7d60-475f-9c8e-761ee2985739 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24090502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24091475Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24092484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24093485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240946961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240957471Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.240977581Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61375)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.241031491Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47478)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.241046852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35535)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.241101402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.241644088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.2417434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.24180848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.241955562Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242040143Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[0]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 254, 254, 254, 254, 250, 254, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242149684Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[0]R generation[0..12]: [144, 146, 144, 146, 144, 148, 148, 148, 148, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242161964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242173775Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[1]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 254, 254, 254, 254, 250, 254, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242187025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[1]R generation[0..12]: [144, 146, 144, 146, 144, 148, 148, 148, 148, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242198025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242209255Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[2]R flush_numbers[0..12]: [247, 252, 246, 252, 246, 254, 254, 254, 254, 250, 254, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242220245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[2]R generation[0..12]: [144, 146, 144, 146, 144, 148, 148, 148, 148, 146, 148, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242231185Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242242325Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242252235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242262386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242272146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242281866Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"":"downstairs","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242292336Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242301556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"0","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242312016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"1","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242322196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"client":"2","":"downstairs","session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242332666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242343566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"e2a71550-7d60-475f-9c8e-761ee2985739 is now active with session: a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242353867Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242363067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"session_id":"a8ce6d7e-260b-4bad-8d1b-9b29dc73d60b"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242372487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Activated sub_volume e2a71550-7d60-475f-9c8e-761ee2985739","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242380787Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242389117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.242820472Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} 01/30 Flush 02/30 Write block 469 len 8192 data: 7 8 03/30 Read block 65 len 12288 04/30 Write block 554 len 24576 data: 13 12 15 18 19 17 05/30 Write block 301 len 12288 data: 16 15 16 06/30 Read block 298 len 4096 07/30 Write block 124 len 4096 data: 8 08/30 Read block 532 len 36864 09/30 Read block 309 len 12288 10/30 Write block 29 len 32768 data: 10 9 8 8 9 10 9 9 11/30 Write block 85 len 40960 data: 11 12 12 10 6 6 9 12 13 12 12/30 Flush 13/30 Write block 126 len 20480 data: 12 13 13 15 13 14/30 Read block 419 len 16384 15/30 Write block 145 len 32768 data: 7 7 7 6 6 5 7 8 16/30 Read block 453 len 32768 17/30 Write block 73 len 28672 data: 15 12 11 11 11 11 11 18/30 Read block 590 len 8192 19/30 Read block 64 len 40960 20/30 Write block 173 len 24576 data: 14 15 14 15 15 18 21/30 Read block 554 len 36864 22/30 Write block 79 len 32768 data: 12 13 14 15 10 10 12 13 23/30 Write block 204 len 4096 data: 10 24/30 Read block 462 len 4096 25/30 Write block 434 len 40960 data: 12 14 14 14 14 15 11 12 12 9 26/30 Write block 194 len 4096 data: 17 27/30 Read block 287 len 32768 28/30 Read block 520 len 32768 29/30 Write block 101 len 12288 data: 16 18 18 30/30 Write block 509 len 28672 data: 12 14 13 13 15 12 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 22 2 2 26 Done 7 27 27 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:41.251572742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:41.251858845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1733,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 146 150 150 252 256 256 F F F <--- 2 040-059 144 144 144 246 246 246 F F F 3 060-079 146 146 146 252 252 252 F T T <--- 4 080-099 144 150 150 246 256 256 F T T <--- 5 100-119 148 148 148 254 254 254 F T T <--- 6 120-139 148 150 150 254 256 256 T T T <--- 7 140-159 148 148 148 254 254 254 F T T <--- 8 160-179 148 148 148 254 254 254 F T T <--- 9 180-199 146 146 146 250 250 250 F T T <--- 10 200-219 148 148 148 254 254 254 F T T <--- 11 220-239 148 148 148 254 254 254 F F F 12 240-259 148 148 148 254 254 254 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 143 150 150 244 256 256 T F F <--- 16 320-339 140 140 140 239 239 239 F F F 17 340-359 148 148 148 254 254 254 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 144 144 144 247 247 247 F T T <--- 22 440-459 144 144 144 247 247 247 F T T <--- 23 460-479 145 150 150 248 256 256 T F F <--- 24 480-499 146 146 146 250 250 250 F F F 25 500-519 148 148 148 254 254 254 F T T <--- 26 520-539 148 148 148 254 254 254 F F F 27 540-559 148 150 150 254 256 256 T F F <--- 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 150, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 150 150 150 256 256 256 F F F 2 040-059 144 144 144 246 246 246 F F F 3 060-079 151 151 151 257 257 257 F F F 4 080-099 151 151 151 257 257 257 F F F 5 100-119 151 151 151 257 257 257 F F F 6 120-139 151 151 151 257 257 257 F F F 7 140-159 151 151 151 257 257 257 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 151 151 151 257 257 257 F F F 10 200-219 151 151 151 257 257 257 F F F 11 220-239 148 148 148 254 254 254 F F F 12 240-259 148 148 148 254 254 254 F F F 13 260-279 145 145 145 248 248 248 F F F 14 280-299 132 132 132 226 226 226 F F F 15 300-319 150 150 150 256 256 256 F F F 16 320-339 140 140 140 239 239 239 F F F 17 340-359 148 148 148 254 254 254 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 137 137 137 234 234 234 F F F 21 420-439 151 151 151 257 257 257 F F F 22 440-459 151 151 151 257 257 257 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 151 151 151 257 257 257 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 143 143 143 244 244 244 F F F 29 580-599 143 143 143 244 244 244 F F F Max gen: 151, Max flush: 257 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.515038772Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.523673321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.523964734Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.523992125Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524001515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524013925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Crucible a650617a-5ad7-4f4e-a8de-4b400bd8ef6c has session id: 4014abb8-0521-4633-9ea0-695652ca6d97","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524406179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Upstairs opts: Upstairs UUID: a650617a-5ad7-4f4e-a8de-4b400bd8ef6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52444551Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Crucible stats registered with UUID: a650617a-5ad7-4f4e-a8de-4b400bd8ef6c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52446027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:45.52446997Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524627782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524644842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524655402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524807054Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524826154Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"a650617a-5ad7-4f4e-a8de-4b400bd8ef6c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524849774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524859634Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524869895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524879965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524890015Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524900365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.524910695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:58801)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525009826Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:60105)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525070887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:59599)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525083687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"io task","client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525692284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525705434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525716434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.525996217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526006198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[0]R flush_numbers[0..12]: [247, 256, 246, 257, 257, 257, 257, 257, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526035838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[0]R generation[0..12]: [144, 150, 144, 151, 151, 151, 151, 151, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526047498Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526058788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[1]R flush_numbers[0..12]: [247, 256, 246, 257, 257, 257, 257, 257, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526069878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[1]R generation[0..12]: [144, 150, 144, 151, 151, 151, 151, 151, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526080808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526092259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[2]R flush_numbers[0..12]: [247, 256, 246, 257, 257, 257, 257, 257, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526103329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[2]R generation[0..12]: [144, 150, 144, 151, 151, 151, 151, 151, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526114259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526127239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526137119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526147259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526157049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526166829Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"":"downstairs","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52617679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52618619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"0","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52619615Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"1","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52620601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"client":"2","":"downstairs","session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52621622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52622577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"a650617a-5ad7-4f4e-a8de-4b400bd8ef6c is now active with session: 4014abb8-0521-4633-9ea0-695652ca6d97","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52623614Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52624532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"session_id":"4014abb8-0521-4633-9ea0-695652ca6d97"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.52625512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Activated sub_volume a650617a-5ad7-4f4e-a8de-4b400bd8ef6c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526263701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526272321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.526674975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} 01/30 Write block 92 len 20480 data: 13 14 13 14 14 02/30 Write block 103 len 8192 data: 19 16 03/30 Read block 459 len 32768 04/30 Read block 238 len 32768 05/30 Flush 06/30 Write block 140 len 28672 data: 6 5 5 6 6 8 8 07/30 Write block 250 len 8192 data: 15 14 08/30 Read block 132 len 24576 09/30 Read block 500 len 20480 10/30 Write block 445 len 4096 data: 7 11/30 Read block 388 len 32768 12/30 Write block 315 len 28672 data: 9 11 12 10 9 9 11 13/30 Read block 200 len 28672 14/30 Flush 15/30 Flush 16/30 Write block 36 len 32768 data: 10 8 8 10 9 10 8 12 17/30 Write block 414 len 4096 data: 11 18/30 Write block 120 len 36864 data: 6 7 7 7 9 9 13 14 14 19/30 Write block 576 len 32768 data: 19 16 18 18 19 19 14 13 20/30 Write block 60 len 16384 data: 9 9 8 7 21/30 Write block 46 len 4096 data: 11 22/30 Read block 29 len 16384 23/30 Write block 416 len 28672 data: 12 11 11 10 8 6 6 24/30 Read block 226 len 36864 25/30 Read block 65 len 28672 26/30 Read block 22 len 28672 27/30 Write block 343 len 32768 data: 16 16 14 14 12 11 12 8 28/30 Read block 513 len 36864 29/30 Write block 279 len 8192 data: 17 13 30/30 Read block 330 len 32768 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:45.534909539Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:45.535210533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1741,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 150 150 150 256 256 256 T F T <--- 2 040-059 144 144 144 246 246 246 T F T <--- 3 060-079 151 151 151 257 257 257 T F T <--- 4 080-099 152 151 152 258 257 258 F F F <--- 5 100-119 152 151 152 258 257 258 F F F <--- 6 120-139 151 151 151 257 257 257 T F T <--- 7 140-159 152 151 152 259 257 259 F F F <--- 8 160-179 151 151 151 257 257 257 F F F 9 180-199 151 151 151 257 257 257 F F F 10 200-219 151 151 151 257 257 257 F F F 11 220-239 148 148 148 254 254 254 F F F 12 240-259 152 148 152 259 254 259 F F F <--- 13 260-279 145 145 145 248 248 248 T F T <--- 14 280-299 132 132 132 226 226 226 T F T <--- 15 300-319 152 150 152 259 256 259 F F F <--- 16 320-339 152 140 152 259 239 259 F F F <--- 17 340-359 148 148 148 254 254 254 T F T <--- 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 137 137 137 234 234 234 T F T <--- 21 420-439 151 151 151 257 257 257 T F T <--- 22 440-459 152 151 152 259 257 259 F F F <--- 23 460-479 150 150 150 256 256 256 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 151 151 151 257 257 257 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 143 143 143 244 244 244 T F T <--- 29 580-599 143 143 143 244 244 244 T F T <--- Max gen: 152, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 153 153 153 260 260 260 F F F 3 060-079 153 153 153 260 260 260 F F F 4 080-099 152 152 152 258 258 258 F F F 5 100-119 152 152 152 258 258 258 F F F 6 120-139 153 153 153 260 260 260 F F F 7 140-159 152 152 152 259 259 259 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 151 151 151 257 257 257 F F F 10 200-219 151 151 151 257 257 257 F F F 11 220-239 148 148 148 254 254 254 F F F 12 240-259 152 152 152 259 259 259 F F F 13 260-279 153 153 153 260 260 260 F F F 14 280-299 153 153 153 260 260 260 F F F 15 300-319 152 152 152 259 259 259 F F F 16 320-339 152 152 152 259 259 259 F F F 17 340-359 153 153 153 260 260 260 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 153 153 153 260 260 260 F F F 21 420-439 153 153 153 260 260 260 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 151 151 151 257 257 257 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 153 153 153 260 260 260 F F F 29 580-599 153 153 153 260 260 260 F F F Max gen: 153, Max flush: 260 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.792785155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:49.800266121Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.800342502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.807687736Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.807949739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.807974199Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.807983029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.807994729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Crucible 1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a has session id: c5c54ad7-11a9-4642-9b1c-2ff27262e9d2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808352674Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Upstairs opts: Upstairs UUID: 1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808394944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Crucible stats registered with UUID: 1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808405944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:49.808415504Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808600766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808634237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808645827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808760618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808787309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808800309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808809729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808819799Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808829819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808839819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808850129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808860469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45860)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.80895538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37283)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.808989561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60513)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809001831Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"io task","client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809575318Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809609068Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809731059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809869771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.809894421Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[0]R flush_numbers[0..12]: [247, 260, 260, 260, 258, 258, 260, 259, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810018683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[0]R generation[0..12]: [144, 153, 153, 153, 152, 152, 153, 152, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810030253Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810041533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[1]R flush_numbers[0..12]: [247, 260, 260, 260, 258, 258, 260, 259, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810052453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[1]R generation[0..12]: [144, 153, 153, 153, 152, 152, 153, 152, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810063413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810074673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[2]R flush_numbers[0..12]: [247, 260, 260, 260, 258, 258, 260, 259, 257, 257, 257, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810085603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[2]R generation[0..12]: [144, 153, 153, 153, 152, 152, 153, 152, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810096344Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810107604Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810117364Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810127244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810138324Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810147984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"":"downstairs","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810158044Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810167124Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"0","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810177064Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"1","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810187035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"client":"2","":"downstairs","session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810197275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810206835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a is now active with session: c5c54ad7-11a9-4642-9b1c-2ff27262e9d2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810217275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810226405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"session_id":"c5c54ad7-11a9-4642-9b1c-2ff27262e9d2"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810235905Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Activated sub_volume 1e1e9a60-61bc-44d4-b8ad-ea18c6ce8b1a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810244155Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.810252465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.81069508Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} 01/30 Write block 204 len 12288 data: 11 8 5 02/30 Read block 84 len 28672 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 258 len 20480 data: 18 17 15 15 18 07/30 Read block 234 len 8192 08/30 Read block 300 len 28672 09/30 Write block 434 len 16384 data: 13 15 15 15 10/30 Read block 378 len 40960 11/30 Read block 158 len 28672 12/30 Read block 548 len 36864 13/30 Write block 286 len 36864 data: 7 8 9 11 11 10 13 12 10 14/30 Read block 491 len 28672 15/30 Flush 16/30 Write block 554 len 4096 data: 14 17/30 Read block 382 len 20480 18/30 Read block 574 len 24576 19/30 Read block 235 len 36864 20/30 Write block 408 len 36864 data: 9 7 6 9 10 10 12 11 13 21/30 Read block 502 len 24576 22/30 Write block 502 len 20480 data: 13 12 11 10 9 23/30 Read block 174 len 12288 24/30 Read block 286 len 8192 25/30 Read block 540 len 8192 26/30 Write block 147 len 12288 data: 8 7 7 27/30 Read block 380 len 24576 28/30 Read block 569 len 36864 29/30 Read block 105 len 36864 30/30 Write block 75 len 24576 data: 12 12 12 12 13 14 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:49.819807084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:49.820089087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1749,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 153 153 153 260 260 260 F F F 3 060-079 153 153 153 260 260 260 F T T <--- 4 080-099 152 152 152 258 258 258 F T T <--- 5 100-119 152 152 152 258 258 258 F F F 6 120-139 153 153 153 260 260 260 F F F 7 140-159 152 152 152 259 259 259 F T T <--- 8 160-179 151 151 151 257 257 257 F F F 9 180-199 151 151 151 257 257 257 F F F 10 200-219 151 154 154 257 261 261 T F F <--- 11 220-239 148 148 148 254 254 254 F F F 12 240-259 152 154 154 259 264 264 F F F <--- 13 260-279 153 154 154 260 264 264 F F F <--- 14 280-299 153 154 154 260 264 264 F F F <--- 15 300-319 152 152 152 259 259 259 F F F 16 320-339 152 152 152 259 259 259 F F F 17 340-359 153 153 153 260 260 260 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 153 153 153 260 260 260 F T T <--- 21 420-439 153 154 154 260 264 264 F F F <--- 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 151 151 151 257 257 257 F T T <--- 26 520-539 148 148 148 254 254 254 F F F 27 540-559 150 150 150 256 256 256 F T T <--- 28 560-579 153 153 153 260 260 260 F F F 29 580-599 153 153 153 260 260 260 F F F Max gen: 154, Max flush: 264 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 153 153 153 260 260 260 F F F 2 040-059 153 153 153 260 260 260 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 155 155 155 265 265 265 F F F 5 100-119 152 152 152 258 258 258 F F F 6 120-139 153 153 153 260 260 260 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 151 151 151 257 257 257 F F F 10 200-219 154 154 154 261 261 261 F F F 11 220-239 148 148 148 254 254 254 F F F 12 240-259 154 154 154 264 264 264 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 F F F 15 300-319 152 152 152 259 259 259 F F F 16 320-339 152 152 152 259 259 259 F F F 17 340-359 153 153 153 260 260 260 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 F F F 20 400-419 155 155 155 265 265 265 F F F 21 420-439 154 154 154 264 264 264 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 146 146 146 250 250 250 F F F 25 500-519 155 155 155 265 265 265 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 155 155 155 265 265 265 F F F 28 560-579 153 153 153 260 260 260 F F F 29 580-599 153 153 153 260 260 260 F F F Max gen: 155, Max flush: 265 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.069761179Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.078707821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079012715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079036935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079048115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079060195Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Crucible f55745ca-078a-4a7b-83f6-fa838f1a3285 has session id: 0b9a991d-7bb6-4fcc-afa8-0584341be3bd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.07942516Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Upstairs opts: Upstairs UUID: f55745ca-078a-4a7b-83f6-fa838f1a3285, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.07946632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Crucible stats registered with UUID: f55745ca-078a-4a7b-83f6-fa838f1a3285","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.07947715Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:54.07948646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079678452Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079694353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079704753Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079844544Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079868845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"f55745ca-078a-4a7b-83f6-fa838f1a3285 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079892425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079902035Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079912095Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079923745Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079933815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.079944426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080006446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:42467)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080040517Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65373)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080068677Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44292)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080079457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"io task","client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080773935Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080843246Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.080947447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081099329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081164619Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[0]R flush_numbers[0..12]: [247, 260, 260, 265, 265, 258, 260, 265, 257, 257, 261, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.08124695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[0]R generation[0..12]: [144, 153, 153, 155, 155, 152, 153, 155, 151, 151, 154, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081258951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081270591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[1]R flush_numbers[0..12]: [247, 260, 260, 265, 265, 258, 260, 265, 257, 257, 261, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081283841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[1]R generation[0..12]: [144, 153, 153, 155, 155, 152, 153, 155, 151, 151, 154, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081294991Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081306211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[2]R flush_numbers[0..12]: [247, 260, 260, 265, 265, 258, 260, 265, 257, 257, 261, 254]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081317131Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[2]R generation[0..12]: [144, 153, 153, 155, 155, 152, 153, 155, 151, 151, 154, 148]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081328111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081339341Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081349162Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081359232Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081368922Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081378662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"":"downstairs","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081388732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081397862Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"0","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081407902Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"1","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081418142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"client":"2","":"downstairs","session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081429693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081439223Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"f55745ca-078a-4a7b-83f6-fa838f1a3285 is now active with session: 0b9a991d-7bb6-4fcc-afa8-0584341be3bd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081449533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081458683Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"session_id":"0b9a991d-7bb6-4fcc-afa8-0584341be3bd"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081468233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Activated sub_volume f55745ca-078a-4a7b-83f6-fa838f1a3285","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081476773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081485343Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.081907438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} 01/30 Write block 233 len 8192 data: 14 14 02/30 Read block 535 len 32768 03/30 Read block 408 len 12288 04/30 Read block 152 len 8192 05/30 Write block 180 len 12288 data: 18 14 13 06/30 Flush 07/30 Write block 129 len 36864 data: 16 14 13 14 17 16 12 10 10 08/30 Read block 170 len 32768 09/30 Write block 480 len 4096 data: 5 10/30 Write block 583 len 24576 data: 14 15 13 13 12 10 11/30 Read block 415 len 8192 12/30 Read block 50 len 20480 13/30 Flush 14/30 Write block 187 len 36864 data: 13 12 11 13 16 19 17 18 15 15/30 Read block 473 len 4096 16/30 Write block 25 len 12288 data: 9 8 7 17/30 Read block 154 len 20480 18/30 Write block 209 len 12288 data: 11 13 15 19/30 Read block 149 len 8192 20/30 Write block 559 len 24576 data: 18 12 12 13 13 12 21/30 Read block 293 len 8192 22/30 Read block 572 len 28672 23/30 Read block 17 len 24576 24/30 Write block 302 len 28672 data: 16 17 16 14 11 9 8 25/30 Flush 26/30 Write block 395 len 40960 data: 9 10 11 13 8 10 10 11 11 14 27/30 Read block 299 len 36864 28/30 Read block 13 len 32768 29/30 Write block 487 len 20480 data: 7 7 8 11 13 30/30 Write block 115 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 2 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 27 31 Done 28 28 3 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:54.090753169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:54.091035012Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1757,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 156 156 153 268 268 260 F F F <--- 2 040-059 153 153 153 260 260 260 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 155 155 155 265 265 265 F F F 5 100-119 152 152 152 258 258 258 T T F <--- 6 120-139 156 156 153 267 267 260 F F F <--- 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 156 156 151 268 268 257 F F F <--- 10 200-219 156 156 154 268 268 261 F F F <--- 11 220-239 156 156 148 266 266 254 F F T <--- 12 240-259 154 154 154 264 264 264 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 F F F 15 300-319 156 156 152 268 268 259 F F F <--- 16 320-339 152 152 152 259 259 259 F F F 17 340-359 153 153 153 260 260 260 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 148 148 148 254 254 254 T T F <--- 20 400-419 155 155 155 265 265 265 T T F <--- 21 420-439 154 154 154 264 264 264 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 156 156 146 267 267 250 T T F <--- 25 500-519 155 155 155 265 265 265 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 156 156 155 268 268 265 F F F <--- 28 560-579 156 156 153 268 268 260 F F F <--- 29 580-599 156 156 153 267 267 260 F F F <--- Max gen: 156, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 156 156 156 268 268 268 F F F 2 040-059 153 153 153 260 260 260 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 155 155 155 265 265 265 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 156 156 156 267 267 267 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 156 156 156 268 268 268 F F F 10 200-219 156 156 156 268 268 268 F F F 11 220-239 156 156 156 266 266 266 F F F 12 240-259 154 154 154 264 264 264 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 F F F 15 300-319 156 156 156 268 268 268 F F F 16 320-339 152 152 152 259 259 259 F F F 17 340-359 153 153 153 260 260 260 F F F 18 360-379 147 147 147 253 253 253 F F F 19 380-399 157 157 157 269 269 269 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 154 154 154 264 264 264 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 F F F 24 480-499 157 157 157 269 269 269 F F F 25 500-519 155 155 155 265 265 265 F F F 26 520-539 148 148 148 254 254 254 F F F 27 540-559 156 156 156 268 268 268 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 157, Max flush: 269 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.336618718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.345312847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.345631021Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.345663311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.345672501Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.345684391Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Crucible a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb has session id: 546666f8-aa24-4b2c-8ef2-6b99d05828d7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346040265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Upstairs opts: Upstairs UUID: a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346071036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Crucible stats registered with UUID: a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346084506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:58.346094126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346280568Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346296768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346307309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34642105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34644704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34646371Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34647299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346483021Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346493181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346503151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346513571Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346523891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38466)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346656093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54060)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346769454Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51876)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.346813954Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"io task","client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34730058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.34733499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347417501Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347635814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347658354Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[0]R flush_numbers[0..12]: [247, 268, 260, 265, 265, 269, 267, 265, 257, 268, 268, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347745895Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[0]R generation[0..12]: [144, 156, 153, 155, 155, 157, 156, 155, 151, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347757915Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347769145Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[1]R flush_numbers[0..12]: [247, 268, 260, 265, 265, 269, 267, 265, 257, 268, 268, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347780105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[1]R generation[0..12]: [144, 156, 153, 155, 155, 157, 156, 155, 151, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347790916Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347802146Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[2]R flush_numbers[0..12]: [247, 268, 260, 265, 265, 269, 267, 265, 257, 268, 268, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347813016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[2]R generation[0..12]: [144, 156, 153, 155, 155, 157, 156, 155, 151, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347826216Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347837516Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347847056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347856976Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347866676Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347876526Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"":"downstairs","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347886567Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347895627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"0","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347905687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"1","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347915577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"client":"2","":"downstairs","session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347925807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347935527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb is now active with session: 546666f8-aa24-4b2c-8ef2-6b99d05828d7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347945937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347955027Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"session_id":"546666f8-aa24-4b2c-8ef2-6b99d05828d7"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347964748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Activated sub_volume a0aa25c8-51d2-4ac0-9c7f-408cfce3a9bb","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347973218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.347982878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.348366412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} 01/30 Write block 545 len 32768 data: 19 21 21 17 14 17 17 16 02/30 Read block 456 len 28672 03/30 Read block 363 len 12288 04/30 Write block 330 len 40960 data: 11 10 14 14 12 9 9 9 9 11 05/30 Read block 2 len 40960 06/30 Read block 26 len 16384 07/30 Flush 08/30 Write block 339 len 16384 data: 12 9 12 13 09/30 Read block 118 len 36864 10/30 Read block 525 len 20480 11/30 Read block 522 len 16384 12/30 Read block 171 len 24576 13/30 Write block 470 len 20480 data: 9 8 9 10 12 14/30 Read block 502 len 16384 15/30 Read block 319 len 8192 16/30 Write block 474 len 28672 data: 13 11 9 8 8 7 6 17/30 Write block 533 len 20480 data: 14 13 14 16 14 18/30 Write block 35 len 24576 data: 10 11 9 9 11 10 19/30 Write block 426 len 16384 data: 8 8 9 9 20/30 Read block 227 len 40960 21/30 Read block 239 len 20480 22/30 Write block 331 len 20480 data: 11 15 15 13 10 23/30 Write block 80 len 32768 data: 15 15 16 11 11 13 14 13 24/30 Write block 197 len 32768 data: 15 16 17 15 15 16 13 12 25/30 Write block 353 len 32768 data: 9 12 12 11 14 15 13 12 26/30 Read block 131 len 28672 27/30 Write block 345 len 12288 data: 15 15 13 28/30 Write block 378 len 32768 data: 11 9 10 10 12 13 13 14 29/30 Write block 520 len 32768 data: 13 11 10 9 10 10 9 7 30/30 Read block 432 len 24576 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) None Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:25:58.356970841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:25:58.357258804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1765,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 156 156 156 268 268 268 T F T <--- 2 040-059 153 153 153 260 260 260 T F T <--- 3 060-079 155 155 155 265 265 265 F F F 4 080-099 155 155 155 265 265 265 T F T <--- 5 100-119 157 157 157 269 269 269 F F F 6 120-139 156 156 156 267 267 267 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 156 156 156 268 268 268 T F T <--- 10 200-219 156 156 156 268 268 268 T F T <--- 11 220-239 156 156 156 266 266 266 F F F 12 240-259 154 154 154 264 264 264 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 F F F 15 300-319 156 156 156 268 268 268 F F F 16 320-339 158 152 158 270 259 270 T F T <--- 17 340-359 153 153 153 260 260 260 T F T <--- 18 360-379 147 147 147 253 253 253 T F T <--- 19 380-399 157 157 157 269 269 269 T F T <--- 20 400-419 157 157 157 269 269 269 F F F 21 420-439 154 154 154 264 264 264 T F T <--- 22 440-459 152 152 152 259 259 259 F F F 23 460-479 150 150 150 256 256 256 T F T <--- 24 480-499 157 157 157 269 269 269 T F T <--- 25 500-519 155 155 155 265 265 265 F F F 26 520-539 148 148 148 254 254 254 T F T <--- 27 540-559 158 156 158 270 268 270 F T F <--- 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 158, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 247 247 247 F F F 1 020-039 159 159 159 271 271 271 F F F 2 040-059 159 159 159 271 271 271 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 159 159 159 271 271 271 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 156 156 156 267 267 267 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 159 159 159 271 271 271 F F F 10 200-219 159 159 159 271 271 271 F F F 11 220-239 156 156 156 266 266 266 F F F 12 240-259 154 154 154 264 264 264 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 F F F 15 300-319 156 156 156 268 268 268 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 159 159 159 271 271 271 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 159 159 159 271 271 271 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 159 159 159 271 271 271 F F F 24 480-499 159 159 159 271 271 271 F F F 25 500-519 155 155 155 265 265 265 F F F 26 520-539 159 159 159 271 271 271 F F F 27 540-559 158 158 158 270 270 270 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 159, Max flush: 271 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.608687136Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617147923Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617437026Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617462156Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617471317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617483317Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Crucible 82553ceb-a02e-466c-b8a3-f8a5c407e116 has session id: b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617854931Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Upstairs opts: Upstairs UUID: 82553ceb-a02e-466c-b8a3-f8a5c407e116, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617884781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Crucible stats registered with UUID: 82553ceb-a02e-466c-b8a3-f8a5c407e116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.617896311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:02.617906172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618098014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618114744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618127684Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618266836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618285956Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"82553ceb-a02e-466c-b8a3-f8a5c407e116 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618319106Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618328846Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618339126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618349067Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618359167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618369487Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618379657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58369)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618414807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43980)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618443628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56662)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.618455148Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"io task","client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619165886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619191886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619206646Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619453869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.61947917Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[0]R flush_numbers[0..12]: [247, 271, 271, 265, 271, 269, 267, 265, 257, 271, 271, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.6195257Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[0]R generation[0..12]: [144, 159, 159, 155, 159, 157, 156, 155, 151, 159, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.61953808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.61954961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[1]R flush_numbers[0..12]: [247, 271, 271, 265, 271, 269, 267, 265, 257, 271, 271, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.6195607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[1]R generation[0..12]: [144, 159, 159, 155, 159, 157, 156, 155, 151, 159, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619571541Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619582901Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[2]R flush_numbers[0..12]: [247, 271, 271, 265, 271, 269, 267, 265, 257, 271, 271, 266]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619594111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[2]R generation[0..12]: [144, 159, 159, 155, 159, 157, 156, 155, 151, 159, 159, 156]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619605081Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619616201Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619625981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619636251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619647311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619657052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"":"downstairs","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619667072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619676322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"0","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619686342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"1","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619696342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"client":"2","":"downstairs","session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619706552Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619715892Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"82553ceb-a02e-466c-b8a3-f8a5c407e116 is now active with session: b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619726242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619735342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"session_id":"b9e7fcd2-aa21-4c36-ae7a-dbef190a4ce3"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619744953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Activated sub_volume 82553ceb-a02e-466c-b8a3-f8a5c407e116","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619753323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.619762063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.620227398Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} 01/30 Write block 242 len 40960 data: 12 9 9 11 13 14 14 13 16 15 02/30 Write block 41 len 8192 data: 11 9 03/30 Read block 230 len 16384 04/30 Read block 324 len 20480 05/30 Flush 06/30 Write block 127 len 12288 data: 15 15 17 07/30 Read block 346 len 28672 08/30 Read block 566 len 4096 09/30 Read block 494 len 28672 10/30 Write block 392 len 20480 data: 14 12 10 10 11 11/30 Write block 128 len 4096 data: 16 12/30 Read block 32 len 36864 13/30 Read block 354 len 12288 14/30 Read block 331 len 32768 15/30 Flush 16/30 Flush 17/30 Write block 17 len 32768 data: 11 10 11 12 13 11 9 8 18/30 Read block 64 len 4096 19/30 Flush 20/30 Write block 483 len 32768 data: 8 8 7 7 8 8 9 12 21/30 Read block 467 len 8192 22/30 Write block 498 len 8192 data: 14 14 23/30 Flush 24/30 Write block 203 len 20480 data: 14 13 9 6 6 25/30 Write block 386 len 12288 data: 14 16 15 26/30 Read block 568 len 8192 27/30 Write block 237 len 32768 data: 11 11 12 8 11 13 10 10 28/30 Read block 411 len 12288 29/30 Write block 284 len 16384 data: 10 9 8 9 30/30 Write block 490 len 20480 data: 13 14 11 10 9 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 3 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 20 2 24 Done 23 5 23 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1004)) Some(JobId(1022)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:02.628571744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:02.628874157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1773,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 144 160 275 247 275 F F F <--- 1 020-039 160 159 160 275 271 275 F F F <--- 2 040-059 160 160 160 272 272 272 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 159 159 159 271 271 271 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 156 160 273 267 273 F T F <--- 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 159 159 159 271 271 271 F F F 10 200-219 159 159 159 271 271 271 T F T <--- 11 220-239 156 156 156 266 266 266 T F T <--- 12 240-259 160 160 160 272 272 272 T F T <--- 13 260-279 154 154 154 264 264 264 F F F 14 280-299 154 154 154 264 264 264 T F T <--- 15 300-319 156 156 156 268 268 268 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 159 159 159 271 271 271 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 160 159 160 273 271 273 T T T <--- 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 159 159 159 271 271 271 F F F 24 480-499 160 159 160 276 271 276 T F T <--- 25 500-519 155 155 155 265 265 265 F F F 26 520-539 159 159 159 271 271 271 F F F 27 540-559 158 158 158 270 270 270 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 160, Max flush: 276 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 275 275 275 F F F 1 020-039 160 160 160 275 275 275 F F F 2 040-059 160 160 160 272 272 272 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 159 159 159 271 271 271 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 159 159 159 271 271 271 F F F 10 200-219 161 161 161 277 277 277 F F F 11 220-239 161 161 161 277 277 277 F F F 12 240-259 161 161 161 277 277 277 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 161 161 161 277 277 277 F F F 15 300-319 156 156 156 268 268 268 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 159 159 159 271 271 271 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 161 161 161 277 277 277 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 159 159 159 271 271 271 F F F 24 480-499 161 161 161 277 277 277 F F F 25 500-519 155 155 155 265 265 265 F F F 26 520-539 159 159 159 271 271 271 F F F 27 540-559 158 158 158 270 270 270 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 161, Max flush: 277 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.874776265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:06.882304361Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.882391852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.889899708Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890178041Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890203842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890213082Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890225022Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Crucible 895c91d3-b15b-48fa-b7c3-fdfd3eb12984 has session id: 3c278180-1ae5-4c83-8c19-5473d6b668a5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890619106Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Upstairs opts: Upstairs UUID: 895c91d3-b15b-48fa-b7c3-fdfd3eb12984, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890647927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Crucible stats registered with UUID: 895c91d3-b15b-48fa-b7c3-fdfd3eb12984","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890659377Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:06.890669017Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890839809Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890858119Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.890868569Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891037291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891064611Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"895c91d3-b15b-48fa-b7c3-fdfd3eb12984 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891078342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891087742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891100392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891110652Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891120742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891131242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891141432Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58953)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891343175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36535)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891367565Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33298)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891378985Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"io task","client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.891983672Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892004512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892026352Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892321356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892331806Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[0]R flush_numbers[0..12]: [275, 275, 272, 265, 271, 269, 273, 265, 257, 271, 277, 277]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892368406Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[0]R generation[0..12]: [160, 160, 160, 155, 159, 157, 160, 155, 151, 159, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892380046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892394157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[1]R flush_numbers[0..12]: [275, 275, 272, 265, 271, 269, 273, 265, 257, 271, 277, 277]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892405327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[1]R generation[0..12]: [160, 160, 160, 155, 159, 157, 160, 155, 151, 159, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892416357Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892427817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[2]R flush_numbers[0..12]: [275, 275, 272, 265, 271, 269, 273, 265, 257, 271, 277, 277]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892439087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[2]R generation[0..12]: [160, 160, 160, 155, 159, 157, 160, 155, 151, 159, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892450327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892461717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892471508Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892481638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892491558Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892501328Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"":"downstairs","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892511468Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892520848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"0","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892530918Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"1","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892541058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"client":"2","":"downstairs","session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892552778Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892562369Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"895c91d3-b15b-48fa-b7c3-fdfd3eb12984 is now active with session: 3c278180-1ae5-4c83-8c19-5473d6b668a5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892572939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892582209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"session_id":"3c278180-1ae5-4c83-8c19-5473d6b668a5"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892591819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Activated sub_volume 895c91d3-b15b-48fa-b7c3-fdfd3eb12984","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892600239Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892608759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.892984063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} 01/30 Flush 02/30 Write block 36 len 8192 data: 12 10 03/30 Read block 375 len 12288 04/30 Write block 341 len 32768 data: 13 14 17 17 16 16 14 12 05/30 Write block 220 len 28672 data: 10 11 12 11 12 16 15 06/30 Read block 13 len 40960 07/30 Flush 08/30 Write block 27 len 28672 data: 8 9 11 10 9 9 10 09/30 Write block 200 len 20480 data: 16 16 17 15 14 10/30 Read block 464 len 28672 11/30 Read block 295 len 36864 12/30 Write block 206 len 40960 data: 7 7 10 12 14 16 16 14 17 15 13/30 Read block 94 len 20480 14/30 Flush 15/30 Write block 475 len 36864 data: 12 10 9 9 8 7 5 6 9 16/30 Read block 29 len 40960 17/30 Read block 357 len 28672 18/30 Flush 19/30 Write block 7 len 36864 data: 10 7 9 10 11 11 10 12 12 20/30 Write block 527 len 32768 data: 8 12 10 10 11 11 15 14 21/30 Write block 480 len 32768 data: 8 6 7 10 9 8 8 9 22/30 Read block 236 len 24576 23/30 Read block 437 len 24576 24/30 Write block 285 len 8192 data: 10 9 25/30 Write block 510 len 36864 data: 15 14 14 16 13 11 11 11 11 26/30 Read block 310 len 32768 27/30 Read block 587 len 12288 28/30 Write block 283 len 4096 data: 8 29/30 Write block 487 len 40960 data: 10 9 10 14 15 12 11 10 8 10 30/30 Read block 494 len 28672 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 7 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 0 21 Done 2 23 23 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:06.902577523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:06.902875237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1781,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 275 275 275 F T T <--- 1 020-039 162 162 162 279 280 280 T F F <--- 2 040-059 160 160 160 272 272 272 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 159 159 159 271 271 271 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 159 159 159 271 271 271 F F F 10 200-219 161 162 162 277 280 280 T F F <--- 11 220-239 162 162 162 279 279 279 F F F 12 240-259 161 161 161 277 277 277 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 161 161 161 277 277 277 F T T <--- 15 300-319 156 156 156 268 268 268 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 162 162 162 279 279 279 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 161 161 161 277 277 277 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 159 162 162 271 281 281 F F F <--- 24 480-499 161 162 162 277 281 281 F T T <--- 25 500-519 155 155 155 265 265 265 F T T <--- 26 520-539 159 159 159 271 271 271 F T T <--- 27 540-559 158 158 158 270 270 270 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 162, Max flush: 281 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 282 282 282 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 160 160 160 272 272 272 F F F 3 060-079 155 155 155 265 265 265 F F F 4 080-099 159 159 159 271 271 271 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 159 159 159 271 271 271 F F F 10 200-219 162 162 162 280 280 280 F F F 11 220-239 162 162 162 279 279 279 F F F 12 240-259 161 161 161 277 277 277 F F F 13 260-279 154 154 154 264 264 264 F F F 14 280-299 163 163 163 282 282 282 F F F 15 300-319 156 156 156 268 268 268 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 162 162 162 279 279 279 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 161 161 161 277 277 277 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 162 162 162 281 281 281 F F F 24 480-499 163 163 163 282 282 282 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 163 163 163 282 282 282 F F F 27 540-559 158 158 158 270 270 270 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 163, Max flush: 282 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.148997228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.157734388Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158044212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158068852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158077872Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158089612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Crucible 14a86bd1-af6e-49c6-81fb-87bdd79d9f31 has session id: 48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158459186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Upstairs opts: Upstairs UUID: 14a86bd1-af6e-49c6-81fb-87bdd79d9f31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158518187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Crucible stats registered with UUID: 14a86bd1-af6e-49c6-81fb-87bdd79d9f31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158530707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:11.158540037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158728739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.15875844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.15877061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158942892Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158968852Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"14a86bd1-af6e-49c6-81fb-87bdd79d9f31 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158981392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.158990662Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159010643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159020773Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159030623Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159045783Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159056173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56373)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159217595Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43838)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159268276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46798)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159282616Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"io task","client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159844052Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.159930063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160008574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160167156Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160210496Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[0]R flush_numbers[0..12]: [282, 280, 272, 265, 271, 269, 273, 265, 257, 271, 280, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160298057Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[0]R generation[0..12]: [163, 162, 160, 155, 159, 157, 160, 155, 151, 159, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160309957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160321518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[1]R flush_numbers[0..12]: [282, 280, 272, 265, 271, 269, 273, 265, 257, 271, 280, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160332698Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[1]R generation[0..12]: [163, 162, 160, 155, 159, 157, 160, 155, 151, 159, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160343718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160355238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[2]R flush_numbers[0..12]: [282, 280, 272, 265, 271, 269, 273, 265, 257, 271, 280, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160366258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[2]R generation[0..12]: [163, 162, 160, 155, 159, 157, 160, 155, 151, 159, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160379438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160390908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160400758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160410879Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160420549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160430509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"":"downstairs","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160440899Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160450189Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"0","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160460659Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"1","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160471039Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"client":"2","":"downstairs","session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160481599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16049137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"14a86bd1-af6e-49c6-81fb-87bdd79d9f31 is now active with session: 48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16050243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16051159Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"session_id":"48fa1f3a-5342-422c-9d4d-25d6b1bc1c0c"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16052147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Activated sub_volume 14a86bd1-af6e-49c6-81fb-87bdd79d9f31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16053018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.16053876Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.160987415Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} 01/30 Write block 546 len 32768 data: 22 22 18 15 18 18 17 14 02/30 Write block 49 len 4096 data: 7 03/30 Read block 434 len 40960 04/30 Read block 72 len 4096 05/30 Write block 14 len 24576 data: 13 13 11 12 11 12 06/30 Write block 226 len 32768 data: 16 16 15 16 15 15 15 15 07/30 Write block 553 len 4096 data: 15 08/30 Write block 479 len 8192 data: 9 9 09/30 Write block 190 len 20480 data: 14 17 20 18 19 10/30 Read block 79 len 12288 11/30 Write block 302 len 20480 data: 17 18 17 15 12 12/30 Write block 228 len 36864 data: 16 17 16 16 16 16 15 13 12 13/30 Read block 456 len 32768 14/30 Write block 272 len 12288 data: 10 13 11 15/30 Write block 165 len 36864 data: 13 13 14 11 12 10 12 12 15 16/30 Read block 263 len 40960 17/30 Flush 18/30 Write block 276 len 40960 data: 14 17 21 18 14 10 10 9 11 11 19/30 Read block 375 len 12288 20/30 Read block 532 len 8192 21/30 Flush 22/30 Write block 95 len 16384 data: 15 15 16 14 23/30 Write block 396 len 12288 data: 12 12 14 24/30 Read block 460 len 16384 25/30 Read block 300 len 12288 26/30 Flush 27/30 Write block 530 len 32768 data: 11 12 12 16 15 15 17 15 28/30 Write block 62 len 16384 data: 9 8 7 12 29/30 Write block 275 len 40960 data: 14 15 18 22 19 15 11 11 10 12 30/30 Write block 214 len 24576 data: 18 16 14 14 13 12 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 16 4 24 Done 26 14 26 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:11.169109248Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:11.169380151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1789,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 163 164 283 282 283 F T F <--- 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 160 164 283 272 283 F T F <--- 3 060-079 155 155 155 265 265 265 T F T <--- 4 080-099 164 159 164 285 271 285 F F F <--- 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 151 164 283 257 283 F F F <--- 9 180-199 164 159 164 283 271 283 F T F <--- 10 200-219 162 162 162 280 280 280 T F T <--- 11 220-239 164 162 164 283 279 283 F T F <--- 12 240-259 161 161 161 277 277 277 F F F 13 260-279 164 154 164 284 264 284 T T T <--- 14 280-299 164 163 164 284 282 284 T F T <--- 15 300-319 164 156 164 283 268 283 F T F <--- 16 320-339 159 159 159 271 271 271 F F F 17 340-359 162 162 162 279 279 279 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 161 164 285 277 285 F F F <--- 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 164 162 164 283 281 283 F T F <--- 24 480-499 164 163 164 283 282 283 F T F <--- 25 500-519 163 163 163 282 282 282 F F F 26 520-539 163 163 163 282 282 282 T F T <--- 27 540-559 164 158 164 283 270 283 F T F <--- 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 164, Max flush: 285 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 283 283 283 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 165 165 165 286 286 286 F F F 4 080-099 164 164 164 285 285 285 F F F 5 100-119 157 157 157 269 269 269 F F F 6 120-139 160 160 160 273 273 273 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 164 164 164 283 283 283 F F F 10 200-219 165 165 165 286 286 286 F F F 11 220-239 164 164 164 283 283 283 F F F 12 240-259 161 161 161 277 277 277 F F F 13 260-279 165 165 165 286 286 286 F F F 14 280-299 165 165 165 286 286 286 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 162 162 162 279 279 279 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 164 164 285 285 285 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 164 164 164 283 283 283 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 165 165 165 286 286 286 F F F 27 540-559 164 164 164 283 283 283 F F F 28 560-579 156 156 156 268 268 268 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 165, Max flush: 286 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.428093817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:15.435541742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.435616873Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.442977797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44323953Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44326517Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443274621Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443286891Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Crucible 3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a has session id: e4c9e635-a5f1-45ac-bd96-ff768c7eeddd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443666175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Upstairs opts: Upstairs UUID: 3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443707425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Crucible stats registered with UUID: 3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443718636Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:15.443728426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443900308Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443916868Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.443927718Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44408373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44410254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44411926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44412879Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44413915Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444149431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444159681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444170501Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444211371Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62682)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444239842Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"ds_connection connected from Ok(127.0.0.1:53111)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444392153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58755)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.444419274Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"io task","client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44494373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44495841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.44496994Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445323184Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445334444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[0]R flush_numbers[0..12]: [283, 280, 283, 286, 285, 269, 273, 265, 283, 283, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445374245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[0]R generation[0..12]: [164, 162, 164, 165, 164, 157, 160, 155, 164, 164, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445386225Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445397995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[1]R flush_numbers[0..12]: [283, 280, 283, 286, 285, 269, 273, 265, 283, 283, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445409315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[1]R generation[0..12]: [164, 162, 164, 165, 164, 157, 160, 155, 164, 164, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445420415Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445431845Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[2]R flush_numbers[0..12]: [283, 280, 283, 286, 285, 269, 273, 265, 283, 283, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445443225Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[2]R generation[0..12]: [164, 162, 164, 165, 164, 157, 160, 155, 164, 164, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445454525Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445466196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445476386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445488366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445498356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445508366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"":"downstairs","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445518766Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445528186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"0","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445538556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"1","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445548567Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"client":"2","":"downstairs","session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445559187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445568917Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a is now active with session: e4c9e635-a5f1-45ac-bd96-ff768c7eeddd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445579557Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445589167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"session_id":"e4c9e635-a5f1-45ac-bd96-ff768c7eeddd"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445598917Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Activated sub_volume 3e1e1df9-5fb4-40dc-aaa0-8f552a6d251a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445607547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.445616187Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.446024532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} 01/30 Write block 240 len 16384 data: 9 12 14 11 02/30 Read block 17 len 4096 03/30 Read block 271 len 8192 04/30 Read block 80 len 12288 05/30 Read block 445 len 12288 06/30 Write block 80 len 40960 data: 16 16 17 12 12 14 15 14 11 7 07/30 Write block 117 len 32768 data: 8 8 7 7 8 8 8 10 08/30 Write block 474 len 4096 data: 14 09/30 Read block 534 len 36864 10/30 Read block 566 len 36864 11/30 Read block 488 len 40960 12/30 Write block 533 len 28672 data: 17 16 16 18 16 16 13 13/30 Read block 188 len 12288 14/30 Read block 98 len 4096 15/30 Read block 347 len 20480 16/30 Flush 17/30 Write block 14 len 20480 data: 14 14 12 13 12 18/30 Write block 266 len 20480 data: 16 15 15 12 10 19/30 Read block 465 len 16384 20/30 Flush 21/30 Write block 346 len 16384 data: 17 15 13 13 22/30 Write block 96 len 24576 data: 16 17 15 13 15 17 23/30 Write block 571 len 20480 data: 8 9 10 14 16 24/30 Write block 429 len 8192 data: 10 14 25/30 Write block 76 len 36864 data: 13 13 13 14 17 17 18 13 13 26/30 Read block 482 len 16384 27/30 Write block 112 len 12288 data: 10 10 8 28/30 Write block 344 len 36864 data: 18 17 18 16 14 14 9 11 9 29/30 Write block 182 len 24576 data: 14 14 12 15 15 14 30/30 Read block 331 len 32768 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 1 22 Done 9 30 29 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:15.454259316Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:15.45454228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1797,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 166 166 283 288 288 F F F <--- 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 165 165 165 286 286 286 F T T <--- 4 080-099 164 166 166 285 287 287 T T T <--- 5 100-119 157 166 166 269 287 287 T T T <--- 6 120-139 160 166 166 273 287 287 T F F <--- 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 164 164 164 283 283 283 F T T <--- 10 200-219 165 165 165 286 286 286 F F F 11 220-239 164 164 164 283 283 283 F F F 12 240-259 161 166 166 277 287 287 T F F <--- 13 260-279 165 166 166 286 288 288 F F F <--- 14 280-299 165 165 165 286 286 286 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 162 162 162 279 279 279 F T T <--- 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 164 164 285 285 285 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 159 159 159 271 271 271 F T T <--- 22 440-459 152 152 152 259 259 259 F F F 23 460-479 164 166 166 283 287 287 T F F <--- 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 165 166 166 286 287 287 F F F <--- 27 540-559 164 164 164 283 283 283 F F F 28 560-579 156 156 156 268 268 268 F T T <--- 29 580-599 156 156 156 267 267 267 F F F Max gen: 166, Max flush: 288 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 288 288 288 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 167 167 167 289 289 289 F F F 4 080-099 167 167 167 289 289 289 F F F 5 100-119 167 167 167 289 289 289 F F F 6 120-139 166 166 166 287 287 287 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 167 167 167 289 289 289 F F F 10 200-219 165 165 165 286 286 286 F F F 11 220-239 164 164 164 283 283 283 F F F 12 240-259 166 166 166 287 287 287 F F F 13 260-279 166 166 166 288 288 288 F F F 14 280-299 165 165 165 286 286 286 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 159 159 159 271 271 271 F F F 17 340-359 167 167 167 289 289 289 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 164 164 285 285 285 F F F 20 400-419 157 157 157 269 269 269 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 166 166 166 287 287 287 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 166 166 166 287 287 287 F F F 27 540-559 164 164 164 283 283 283 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 167, Max flush: 289 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.717379502Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726214863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726524757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726550157Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726559437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726571777Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Crucible 2ca7f1a1-63e7-408a-8d18-8414f5799046 has session id: 00e1335e-1440-48a5-8c70-d7ccece8cd33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.726945752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Upstairs opts: Upstairs UUID: 2ca7f1a1-63e7-408a-8d18-8414f5799046, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727000792Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Crucible stats registered with UUID: 2ca7f1a1-63e7-408a-8d18-8414f5799046","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727012202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:19.727028313Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727216455Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727232605Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727243115Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727383397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727400577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"2ca7f1a1-63e7-408a-8d18-8414f5799046 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727438507Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727450057Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727460348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727470518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727497768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727522898Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727534208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"ds_connection connected from Ok(127.0.0.1:53091)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727549629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"ds_connection connected from Ok(127.0.0.1:35215)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.727570689Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"ds_connection connected from Ok(127.0.0.1:61646)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.72771053Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"io task","client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728178776Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728215196Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728373618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728487749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.72851249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[0]R flush_numbers[0..12]: [288, 280, 283, 289, 289, 289, 287, 265, 283, 289, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728717342Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[0]R generation[0..12]: [166, 162, 164, 167, 167, 167, 166, 155, 164, 167, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728730802Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728742252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[1]R flush_numbers[0..12]: [288, 280, 283, 289, 289, 289, 287, 265, 283, 289, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728753392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[1]R generation[0..12]: [166, 162, 164, 167, 167, 167, 166, 155, 164, 167, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728764483Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728775893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[2]R flush_numbers[0..12]: [288, 280, 283, 289, 289, 289, 287, 265, 283, 289, 286, 283]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728787093Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[2]R generation[0..12]: [166, 162, 164, 167, 167, 167, 166, 155, 164, 167, 165, 164]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728798243Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728809573Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728819523Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728829723Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728839643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728849733Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"":"downstairs","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728859924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728869214Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"0","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728880664Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"1","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728890744Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"client":"2","":"downstairs","session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728901034Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728910584Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"2ca7f1a1-63e7-408a-8d18-8414f5799046 is now active with session: 00e1335e-1440-48a5-8c70-d7ccece8cd33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728921034Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728930164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"session_id":"00e1335e-1440-48a5-8c70-d7ccece8cd33"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728939835Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pidDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":1805} {"msg":"Activated sub_volume 2ca7f1a1-63e7-408a-8d18-8414f5799046","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728952545Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.728961215Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.729326999Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} 01/30 Write block 569 len 32768 data: 10 7 9 10 11 15 17 20 02/30 Read block 358 len 8192 03/30 Write block 271 len 32768 data: 11 11 14 12 15 16 19 23 04/30 Flush 05/30 Write block 410 len 40960 data: 7 10 11 11 13 12 14 12 12 11 06/30 Read block 164 len 24576 07/30 Write block 76 len 32768 data: 14 14 14 15 18 18 19 14 08/30 Write block 415 len 20480 data: 13 15 13 13 12 09/30 Read block 116 len 20480 10/30 Write block 418 len 20480 data: 14 13 9 7 7 11/30 Read block 405 len 8192 12/30 Read block 145 len 8192 13/30 Read block 477 len 4096 14/30 Read block 314 len 24576 15/30 Read block 589 len 36864 16/30 Read block 411 len 8192 17/30 Flush 18/30 Write block 470 len 28672 data: 10 9 10 11 15 13 11 19/30 Flush 20/30 Write block 552 len 28672 data: 18 16 15 13 16 19 20 21/30 Write block 230 len 36864 data: 17 17 17 17 16 14 13 12 12 22/30 Write block 259 len 32768 data: 18 16 16 19 20 17 19 17 23/30 Read block 453 len 12288 24/30 Read block 286 len 20480 25/30 Read block 500 len 24576 26/30 Write block 537 len 4096 data: 17 27/30 Write block 321 len 40960 data: 12 12 11 10 11 10 11 12 11 12 28/30 Write block 261 len 4096 data: 17 29/30 Write block 189 len 16384 data: 12 15 18 21 30/30 Write block 5 len 28672 data: 7 10 11 8 10 11 12 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 2 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 10 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 29 37 Done 26 26 1 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:19.737923217Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:19.73818529Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1805,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 288 288 288 T T F <--- 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 168 168 167 291 291 289 F F F <--- 4 080-099 168 168 167 291 291 289 F F F <--- 5 100-119 167 167 167 289 289 289 F F F 6 120-139 166 166 166 287 287 287 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 167 167 167 289 289 289 T T F <--- 10 200-219 165 165 165 286 286 286 F F F 11 220-239 164 164 164 283 283 283 T T F <--- 12 240-259 166 166 166 287 287 287 T T F <--- 13 260-279 168 168 166 290 290 288 T T F <--- 14 280-299 165 165 165 286 286 286 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 159 159 159 271 271 271 T T F <--- 17 340-359 167 167 167 289 289 289 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 164 164 285 285 285 F F F 20 400-419 168 168 157 291 291 269 F F F <--- 21 420-439 168 168 167 291 291 289 F F F <--- 22 440-459 152 152 152 259 259 259 F F F 23 460-479 168 168 166 292 292 287 F F F <--- 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 166 166 166 287 287 287 T T F <--- 27 540-559 164 164 164 283 283 283 T T F <--- 28 560-579 168 168 167 290 290 289 F F T <--- 29 580-599 156 156 156 267 267 267 F F F Max gen: 168, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 293 293 293 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 167 167 167 289 289 289 F F F 6 120-139 166 166 166 287 287 287 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 165 165 165 286 286 286 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 169 169 169 293 293 293 F F F 13 260-279 169 169 169 293 293 293 F F F 14 280-299 165 165 165 286 286 286 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 169 169 169 293 293 293 F F F 17 340-359 167 167 167 289 289 289 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 164 164 164 285 285 285 F F F 20 400-419 168 168 168 291 291 291 F F F 21 420-439 168 168 168 291 291 291 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 168 168 168 292 292 292 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 169 169 169 293 293 293 F F F 27 540-559 169 169 169 293 293 293 F F F 28 560-579 168 168 168 290 290 290 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 169, Max flush: 293 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.990930137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.999472054Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.999772888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.999797768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.999806928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:23.999818888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Crucible bae20377-a94c-4154-9d8d-565f31403903 has session id: e74d5e6c-9df9-4b2b-bb1b-e057188681bc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000394365Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Upstairs opts: Upstairs UUID: bae20377-a94c-4154-9d8d-565f31403903, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000427555Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Crucible stats registered with UUID: bae20377-a94c-4154-9d8d-565f31403903","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000438655Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:24.000449506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000621238Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000638828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000649438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00080703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.0008265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"bae20377-a94c-4154-9d8d-565f31403903 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00085071Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00086043Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00087055Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00088076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000890671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000901231Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000913151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41306)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.000966841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52277)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.001033262Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59585)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.001061703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"io task","client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00172387Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.00174554Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.001783801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002103394Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002127825Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[0]R flush_numbers[0..12]: [293, 280, 283, 291, 291, 289, 287, 265, 283, 293, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002208886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[0]R generation[0..12]: [169, 162, 164, 168, 168, 167, 166, 155, 164, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002221236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002232856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[1]R flush_numbers[0..12]: [293, 280, 283, 291, 291, 289, 287, 265, 283, 293, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002243956Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[1]R generation[0..12]: [169, 162, 164, 168, 168, 167, 166, 155, 164, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002255126Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002268556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[2]R flush_numbers[0..12]: [293, 280, 283, 291, 291, 289, 287, 265, 283, 293, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002279716Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[2]R generation[0..12]: [169, 162, 164, 168, 168, 167, 166, 155, 164, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002290627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002301887Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002311707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002321877Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002331657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002341577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"":"downstairs","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002351857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002361137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"0","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002371478Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"1","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002381628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"client":"2","":"downstairs","session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002392018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002401618Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"bae20377-a94c-4154-9d8d-565f31403903 is now active with session: e74d5e6c-9df9-4b2b-bb1b-e057188681bc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002412088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002422658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"session_id":"e74d5e6c-9df9-4b2b-bb1b-e057188681bc"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002432158Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Activated sub_volume bae20377-a94c-4154-9d8d-565f31403903","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002440728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002449428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.002844063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} 01/30 Flush 02/30 Write block 577 len 4096 data: 17 03/30 Read block 215 len 16384 04/30 Write block 113 len 20480 data: 11 9 8 8 9 05/30 Read block 227 len 40960 06/30 Read block 84 len 36864 07/30 Read block 231 len 36864 08/30 Read block 166 len 32768 09/30 Write block 344 len 8192 data: 19 18 10/30 Write block 203 len 8192 data: 16 15 11/30 Write block 385 len 20480 data: 15 15 17 16 13 12/30 Read block 353 len 28672 13/30 Read block 104 len 32768 14/30 Read block 259 len 32768 15/30 Read block 449 len 12288 16/30 Write block 118 len 24576 data: 9 8 8 9 9 9 17/30 Read block 77 len 12288 18/30 Read block 130 len 4096 19/30 Read block 183 len 36864 20/30 Flush 21/30 Write block 87 len 24576 data: 15 12 8 7 10 14 22/30 Read block 202 len 28672 23/30 Read block 528 len 40960 24/30 Write block 415 len 16384 data: 14 16 14 15 25/30 Read block 213 len 28672 26/30 Write block 297 len 4096 data: 12 27/30 Read block 366 len 4096 28/30 Read block 125 len 36864 29/30 Read block 299 len 40960 30/30 Read block 420 len 8192 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1000)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:24.011208359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:24.011485802Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1813,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 293 293 293 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 T F T <--- 5 100-119 170 167 170 295 289 295 F F F <--- 6 120-139 170 166 170 295 287 295 F F F <--- 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 170 165 170 295 286 295 F F F <--- 11 220-239 169 169 169 293 293 293 F F F 12 240-259 169 169 169 293 293 293 F F F 13 260-279 169 169 169 293 293 293 F F F 14 280-299 165 165 165 286 286 286 T F T <--- 15 300-319 164 164 164 283 283 283 F F F 16 320-339 169 169 169 293 293 293 F F F 17 340-359 170 167 170 295 289 295 F F F <--- 18 360-379 159 159 159 271 271 271 F F F 19 380-399 170 164 170 295 285 295 F F F <--- 20 400-419 168 168 168 291 291 291 T F T <--- 21 420-439 168 168 168 291 291 291 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 168 168 168 292 292 292 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 169 169 169 293 293 293 F F F 27 540-559 169 169 169 293 293 293 F F F 28 560-579 170 168 170 295 290 295 F F F <--- 29 580-599 156 156 156 267 267 267 F F F Max gen: 170, Max flush: 295 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 293 293 293 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 170 170 170 295 295 295 F F F 6 120-139 170 170 170 295 295 295 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 170 170 170 295 295 295 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 169 169 169 293 293 293 F F F 13 260-279 169 169 169 293 293 293 F F F 14 280-299 171 171 171 296 296 296 F F F 15 300-319 164 164 164 283 283 283 F F F 16 320-339 169 169 169 293 293 293 F F F 17 340-359 170 170 170 295 295 295 F F F 18 360-379 159 159 159 271 271 271 F F F 19 380-399 170 170 170 295 295 295 F F F 20 400-419 171 171 171 296 296 296 F F F 21 420-439 168 168 168 291 291 291 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 168 168 168 292 292 292 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 169 169 169 293 293 293 F F F 27 540-559 169 169 169 293 293 293 F F F 28 560-579 170 170 170 295 295 295 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 171, Max flush: 296 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.251991759Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:28.259464594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.259537675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.266725438Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267001551Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267026591Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267035471Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267047481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Crucible 669e2f3f-fbdf-44a7-b1cc-493fcb34c947 has session id: c5036a4a-5791-4e6a-b07c-af1fe6eaa65e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267433986Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Upstairs opts: Upstairs UUID: 669e2f3f-fbdf-44a7-b1cc-493fcb34c947, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267463026Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Crucible stats registered with UUID: 669e2f3f-fbdf-44a7-b1cc-493fcb34c947","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267474506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:28.267486476Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267670168Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267685949Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267696339Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.26783903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267856841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"669e2f3f-fbdf-44a7-b1cc-493fcb34c947 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267893031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267902781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267912941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267922871Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267933051Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267943382Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.267953712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51919)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.268020192Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62503)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.268031553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65372)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.268051163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"io task","client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.26871104Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.268760251Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.268794191Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269057074Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269082025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[0]R flush_numbers[0..12]: [293, 280, 283, 291, 296, 295, 295, 265, 283, 293, 295, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269147615Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[0]R generation[0..12]: [169, 162, 164, 168, 171, 170, 170, 155, 164, 169, 170, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269160005Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269171836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[1]R flush_numbers[0..12]: [293, 280, 283, 291, 296, 295, 295, 265, 283, 293, 295, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269183226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[1]R generation[0..12]: [169, 162, 164, 168, 171, 170, 170, 155, 164, 169, 170, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269194346Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269205876Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[2]R flush_numbers[0..12]: [293, 280, 283, 291, 296, 295, 295, 265, 283, 293, 295, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269216926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[2]R generation[0..12]: [169, 162, 164, 168, 171, 170, 170, 155, 164, 169, 170, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269228026Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269239356Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269251607Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269261957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269271657Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269281457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"":"downstairs","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269291617Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269300897Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"0","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269311087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"1","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269321037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"client":"2","":"downstairs","session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269331247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269340838Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"669e2f3f-fbdf-44a7-b1cc-493fcb34c947 is now active with session: c5036a4a-5791-4e6a-b07c-af1fe6eaa65e","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269351488Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269360658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"session_id":"c5036a4a-5791-4e6a-b07c-af1fe6eaa65e"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269370328Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Activated sub_volume 669e2f3f-fbdf-44a7-b1cc-493fcb34c947","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269378738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269387348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.269786333Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} 01/30 Write block 300 len 4096 data: 14 02/30 Flush 03/30 Write block 323 len 8192 data: 12 11 04/30 Write block 356 len 20480 data: 12 15 16 14 13 05/30 Read block 99 len 36864 06/30 Write block 202 len 36864 data: 18 17 16 10 8 8 11 13 15 07/30 Read block 237 len 24576 08/30 Write block 121 len 8192 data: 10 10 09/30 Read block 14 len 28672 10/30 Read block 193 len 4096 11/30 Write block 435 len 40960 data: 16 16 16 15 16 12 13 13 10 9 12/30 Read block 52 len 24576 13/30 Write block 376 len 24576 data: 10 9 12 10 11 11 14/30 Read block 366 len 12288 15/30 Write block 107 len 12288 data: 12 11 14 16/30 Write block 46 len 20480 data: 12 10 9 8 8 17/30 Write block 528 len 40960 data: 13 11 12 13 13 18 17 17 19 18 18/30 Read block 216 len 4096 19/30 Read block 169 len 32768 20/30 Write block 408 len 40960 data: 10 8 8 11 12 12 14 15 17 15 21/30 Write block 416 len 16384 data: 18 16 16 14 22/30 Read block 116 len 16384 23/30 Write block 337 len 36864 data: 10 10 13 10 14 15 18 20 19 24/30 Read block 100 len 20480 25/30 Read block 242 len 12288 26/30 Write block 354 len 28672 data: 13 13 13 16 17 15 14 27/30 Read block 322 len 28672 28/30 Write block 255 len 40960 data: 12 15 13 19 19 17 18 20 21 18 29/30 Write block 386 len 8192 data: 16 18 30/30 Read block 523 len 32768 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:28.278885227Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:28.2791784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1821,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 293 293 293 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 164 164 164 283 283 283 F T T <--- 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 170 170 170 295 295 295 F T T <--- 6 120-139 170 170 170 295 295 295 F T T <--- 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 170 170 170 295 295 295 F T T <--- 11 220-239 169 169 169 293 293 293 F F F 12 240-259 169 169 169 293 293 293 F T T <--- 13 260-279 169 169 169 293 293 293 F T T <--- 14 280-299 171 171 171 296 296 296 F F F 15 300-319 172 172 172 297 297 297 F F F 16 320-339 169 169 169 293 293 293 T T T 17 340-359 170 170 170 295 295 295 F T T <--- 18 360-379 159 159 159 271 271 271 F T T <--- 19 380-399 170 170 170 295 295 295 F T T <--- 20 400-419 171 171 171 296 296 296 F T T <--- 21 420-439 168 168 168 291 291 291 F T T <--- 22 440-459 152 152 152 259 259 259 F T T <--- 23 460-479 168 168 168 292 292 292 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 169 169 169 293 293 293 F T T <--- 27 540-559 169 169 169 293 293 293 F F F 28 560-579 170 170 170 295 295 295 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 172, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 340,342d339 < 9, < 9, < 12, 343a341,343 > 10, > 13, > 10, diff found after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 293 293 293 F F F 1 020-039 162 162 162 280 280 280 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 173 173 173 298 298 298 F F F 6 120-139 173 173 173 298 298 298 F F F 7 140-159 155 155 155 265 265 265 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 173 173 173 298 298 298 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 173 173 173 298 298 298 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 171 171 171 296 296 296 F F F 15 300-319 172 172 172 297 297 297 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 173 173 173 298 298 298 F F F 19 380-399 173 173 173 298 298 298 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 168 168 168 292 292 292 F F F 24 480-499 164 164 164 283 283 283 F F F 25 500-519 163 163 163 282 282 282 F F F 26 520-539 173 173 173 298 298 298 F F F 27 540-559 169 169 169 293 293 293 F F F 28 560-579 170 170 170 295 295 295 F F F 29 580-599 156 156 156 267 267 267 F F F Max gen: 173, Max flush: 298 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.546628416Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555154193Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555455727Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555478367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555487457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555499397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Crucible f044b1c2-a35c-44fb-8797-594d6786d234 has session id: 08594d5e-7487-4417-ba2a-5fc3eab6b45b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555852821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Upstairs opts: Upstairs UUID: f044b1c2-a35c-44fb-8797-594d6786d234, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555882512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Crucible stats registered with UUID: f044b1c2-a35c-44fb-8797-594d6786d234","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.555893682Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:32.555906522Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556099834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556115954Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556126684Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556259406Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556278156Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"f044b1c2-a35c-44fb-8797-594d6786d234 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556304756Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556314366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556324407Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556334327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556344337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556354907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556365087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49233)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556419858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63259)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556432528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33066)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.556501149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557163836Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557250447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557341608Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.55746738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557542451Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[0]R flush_numbers[0..12]: [293, 280, 298, 291, 296, 298, 298, 265, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557649102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[0]R generation[0..12]: [169, 162, 173, 168, 171, 173, 173, 155, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557660932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557672352Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[1]R flush_numbers[0..12]: [293, 280, 298, 291, 296, 298, 298, 265, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557683132Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[1]R generation[0..12]: [169, 162, 173, 168, 171, 173, 173, 155, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557694002Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557705152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[2]R flush_numbers[0..12]: [293, 280, 298, 291, 296, 298, 298, 265, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557716183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[2]R generation[0..12]: [169, 162, 173, 168, 171, 173, 173, 155, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557726983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557738103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557747793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557757833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557767633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557777323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"":"downstairs","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557788883Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557798103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"0","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557808474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"1","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557818724Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"client":"2","":"downstairs","session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557829004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557838564Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"f044b1c2-a35c-44fb-8797-594d6786d234 is now active with session: 08594d5e-7487-4417-ba2a-5fc3eab6b45b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557848844Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557858004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"session_id":"08594d5e-7487-4417-ba2a-5fc3eab6b45b"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557867424Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Activated sub_volume f044b1c2-a35c-44fb-8797-594d6786d234","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557875704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.557884144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.558286839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} 01/30 Write block 18 len 20480 data: 13 13 13 14 12 02/30 Read block 261 len 40960 03/30 Read block 182 len 8192 04/30 Write block 540 len 4096 data: 15 05/30 Flush 06/30 Write block 135 len 36864 data: 13 11 11 11 9 7 6 6 7 07/30 Read block 237 len 20480 08/30 Flush 09/30 Write block 522 len 32768 data: 11 10 11 11 10 9 14 12 10/30 Read block 162 len 8192 11/30 Read block 129 len 8192 12/30 Write block 476 len 28672 data: 12 10 10 10 10 7 8 13/30 Read block 371 len 28672 14/30 Write block 125 len 8192 data: 10 14 15/30 Write block 20 len 12288 data: 14 15 13 16/30 Read block 86 len 40960 17/30 Read block 251 len 4096 18/30 Read block 412 len 8192 19/30 Read block 396 len 36864 20/30 Write block 104 len 40960 data: 17 19 18 13 12 15 11 12 11 12 21/30 Read block 166 len 16384 22/30 Flush 23/30 Write block 584 len 36864 data: 16 14 14 13 11 9 10 8 7 24/30 Read block 435 len 36864 25/30 Read block 327 len 4096 26/30 Flush 27/30 Write block 506 len 4096 data: 10 28/30 Read block 100 len 28672 29/30 Read block 366 len 40960 30/30 Write block 385 len 28672 data: 16 17 19 17 14 13 14 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:32.567402173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:32.567693117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1832,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 169 174 299 293 299 F T F <--- 1 020-039 174 162 174 301 280 301 F T F <--- 2 040-059 173 173 173 298 298 298 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 174 173 174 301 298 301 F F F <--- 6 120-139 174 173 174 301 298 301 F F F <--- 7 140-159 174 155 174 300 265 300 F F F <--- 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 173 173 173 298 298 298 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 173 173 173 298 298 298 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 171 171 171 296 296 296 F F F 15 300-319 172 172 172 297 297 297 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 173 173 173 298 298 298 F F F 19 380-399 173 173 173 298 298 298 T F T <--- 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 174 168 174 301 292 301 F F F <--- 24 480-499 174 164 174 301 283 301 F F F <--- 25 500-519 163 163 163 282 282 282 T F T <--- 26 520-539 174 173 174 301 298 301 F F F <--- 27 540-559 174 169 174 299 293 299 F F F <--- 28 560-579 170 170 170 295 295 295 F F F 29 580-599 174 156 174 302 267 302 F F F <--- Max gen: 174, Max flush: 302 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 299 299 299 F F F 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 174 174 174 301 301 301 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 174 174 174 300 300 300 F F F 8 160-179 164 164 164 283 283 283 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 173 173 173 298 298 298 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 173 173 173 298 298 298 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 171 171 171 296 296 296 F F F 15 300-319 172 172 172 297 297 297 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 173 173 173 298 298 298 F F F 19 380-399 175 175 175 303 303 303 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 174 174 174 301 301 301 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 175 175 175 303 303 303 F F F 26 520-539 174 174 174 301 301 301 F F F 27 540-559 174 174 174 299 299 299 F F F 28 560-579 170 170 170 295 295 295 F F F 29 580-599 174 174 174 302 302 302 F F F Max gen: 175, Max flush: 303 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.808145893Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:36.815649209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.81572334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823050743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823300566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823323446Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823334986Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823346816Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Crucible c1704995-9053-4227-8899-c010f4a55d3f has session id: fb402b85-6358-4078-bb2f-09d30b9affa1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82364785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Upstairs opts: Upstairs UUID: c1704995-9053-4227-8899-c010f4a55d3f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.8236795Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Crucible stats registered with UUID: c1704995-9053-4227-8899-c010f4a55d3f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82369182Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:36.8237024Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823896653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823926703Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.823937903Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824051294Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824076205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"c1704995-9053-4227-8899-c010f4a55d3f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824087625Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824097045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824107205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824119315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824129265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824139695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824154385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36770)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824173966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49681)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824212936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42579)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824239076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824899714Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824916804Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.824928224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825292948Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825317209Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[0]R flush_numbers[0..12]: [299, 301, 298, 291, 296, 301, 301, 300, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825331049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[0]R generation[0..12]: [174, 174, 173, 168, 171, 174, 174, 174, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825342559Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825354079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[1]R flush_numbers[0..12]: [299, 301, 298, 291, 296, 301, 301, 300, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825367309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[1]R generation[0..12]: [174, 174, 173, 168, 171, 174, 174, 174, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825378259Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82538957Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[2]R flush_numbers[0..12]: [299, 301, 298, 291, 296, 301, 301, 300, 283, 293, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82540075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[2]R generation[0..12]: [174, 174, 173, 168, 171, 174, 174, 174, 164, 169, 173, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82541174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82542291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82543275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82544307Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.8254527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.82546285Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"":"downstairs","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825473031Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825482021Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"0","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825492211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"1","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825502311Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"client":"2","":"downstairs","session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825513731Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825523321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"c1704995-9053-4227-8899-c010f4a55d3f is now active with session: fb402b85-6358-4078-bb2f-09d30b9affa1","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825533721Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825542811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"session_id":"fb402b85-6358-4078-bb2f-09d30b9affa1"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825552321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Activated sub_volume c1704995-9053-4227-8899-c010f4a55d3f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825560762Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825569172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.825950926Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} 01/30 Write block 505 len 28672 data: 11 11 10 11 13 16 15 02/30 Write block 577 len 40960 data: 18 19 19 20 20 15 15 17 15 15 03/30 Write block 525 len 8192 data: 12 11 04/30 Read block 508 len 16384 05/30 Read block 515 len 16384 06/30 Read block 45 len 8192 07/30 Write block 566 len 16384 data: 15 14 15 11 08/30 Read block 325 len 24576 09/30 Flush 10/30 Flush 11/30 Write block 2 len 36864 data: 7 8 7 8 11 12 9 11 12 12/30 Write block 174 len 4096 data: 16 13/30 Read block 43 len 4096 14/30 Read block 110 len 4096 15/30 Flush 16/30 Write block 252 len 20480 data: 14 14 11 13 16 17/30 Write block 205 len 16384 data: 11 9 9 12 18/30 Write block 467 len 40960 data: 5 5 8 11 10 11 12 16 14 13 19/30 Read block 254 len 36864 20/30 Read block 552 len 20480 21/30 Flush 22/30 Write block 105 len 20480 data: 20 19 14 13 16 23/30 Flush 24/30 Write block 543 len 32768 data: 15 16 20 23 23 19 16 19 25/30 Write block 286 len 36864 data: 10 10 10 12 12 11 14 13 11 26/30 Read block 203 len 16384 27/30 Write block 367 len 8192 data: 9 9 28/30 Write block 573 len 28672 data: 12 16 18 21 19 20 20 29/30 Read block 229 len 36864 30/30 Write block 374 len 32768 data: 10 10 11 10 13 11 12 12 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 17 1 1 19 Done 3 19 19 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:36.83416167Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:36.834440693Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1840,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 176 176 299 306 306 T F F <--- 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 174 176 176 301 308 308 F F F <--- 6 120-139 174 174 174 301 301 301 F F F 7 140-159 174 174 174 300 300 300 F F F 8 160-179 164 176 176 283 306 306 T F F <--- 9 180-199 169 169 169 293 293 293 F F F 10 200-219 173 176 176 298 307 307 F F F <--- 11 220-239 169 169 169 293 293 293 F F F 12 240-259 173 176 176 298 307 307 F F F <--- 13 260-279 173 173 173 298 298 298 F F F 14 280-299 171 171 171 296 296 296 F T T <--- 15 300-319 172 172 172 297 297 297 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 173 173 173 298 298 298 F T T <--- 19 380-399 175 175 175 303 303 303 F T T <--- 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 174 176 176 301 307 307 F F F <--- 24 480-499 174 174 174 301 301 301 F F F 25 500-519 176 176 176 304 304 304 F F F 26 520-539 176 176 176 304 304 304 F F F 27 540-559 174 174 174 299 299 299 F T T <--- 28 560-579 176 176 176 304 304 304 F T T <--- 29 580-599 176 176 176 304 304 304 F F F Max gen: 176, Max flush: 308 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 306 306 306 F F F 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 171 171 171 296 296 296 F F F 5 100-119 176 176 176 308 308 308 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 174 174 174 300 300 300 F F F 8 160-179 176 176 176 306 306 306 F F F 9 180-199 169 169 169 293 293 293 F F F 10 200-219 176 176 176 307 307 307 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 176 176 176 307 307 307 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 177 177 177 309 309 309 F F F 15 300-319 172 172 172 297 297 297 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 176 176 176 304 304 304 F F F 26 520-539 176 176 176 304 304 304 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 177 177 177 309 309 309 F F F 29 580-599 176 176 176 304 304 304 F F F Max gen: 177, Max flush: 309 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.088646397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.097303516Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09759499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09761941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09762863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09764038Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Crucible 57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9 has session id: f19f5353-105d-479d-8d2f-acf34d6563b7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098011244Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Upstairs opts: Upstairs UUID: 57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098054785Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Crucible stats registered with UUID: 57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098066045Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:41.098077705Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098246937Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098276367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098287777Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098433679Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098452409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09848771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09849801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09850811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09851828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09852833Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09853867Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64178)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098564841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47409)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098613811Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49043)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.098687322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099289089Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.09935423Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099433911Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099592912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099662643Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[0]R flush_numbers[0..12]: [306, 301, 298, 291, 296, 308, 301, 300, 306, 293, 307, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099731644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[0]R generation[0..12]: [176, 174, 173, 168, 171, 176, 174, 174, 176, 169, 176, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099744754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099756524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[1]R flush_numbers[0..12]: [306, 301, 298, 291, 296, 308, 301, 300, 306, 293, 307, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099767594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[1]R generation[0..12]: [176, 174, 173, 168, 171, 176, 174, 174, 176, 169, 176, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099778685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099790175Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[2]R flush_numbers[0..12]: [306, 301, 298, 291, 296, 308, 301, 300, 306, 293, 307, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099801265Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[2]R generation[0..12]: [176, 174, 173, 168, 171, 176, 174, 174, 176, 169, 176, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099812285Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099823695Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099836435Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099846635Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099856425Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099866366Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"":"downstairs","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099876556Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099885856Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"0","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099895906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"1","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099905796Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"client":"2","":"downstairs","session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099916086Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099925506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9 is now active with session: f19f5353-105d-479d-8d2f-acf34d6563b7","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099935936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099945046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"session_id":"f19f5353-105d-479d-8d2f-acf34d6563b7"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099954457Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Activated sub_volume 57e98a6a-e7bd-4d7f-92ef-b71592c1dfa9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099962927Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.099971387Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.100420072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} 01/30 Write block 147 len 12288 data: 9 8 8 02/30 Flush 03/30 Write block 206 len 8192 data: 10 10 04/30 Read block 67 len 40960 05/30 Read block 418 len 24576 06/30 Read block 521 len 36864 07/30 Read block 421 len 40960 08/30 Read block 470 len 16384 09/30 Read block 131 len 12288 10/30 Write block 573 len 40960 data: 13 17 19 22 20 21 21 21 21 16 11/30 Read block 274 len 4096 12/30 Read block 435 len 20480 13/30 Write block 311 len 24576 data: 12 10 10 9 10 12 14/30 Write block 79 len 16384 data: 16 19 19 20 15/30 Read block 482 len 40960 16/30 Read block 523 len 16384 17/30 Read block 329 len 40960 18/30 Write block 526 len 24576 data: 12 10 15 13 13 14 19/30 Write block 280 len 36864 data: 16 12 12 11 13 12 11 11 11 20/30 Read block 379 len 20480 21/30 Write block 178 len 20480 data: 19 17 19 15 15 22/30 Read block 512 len 16384 23/30 Read block 390 len 40960 24/30 Flush 25/30 Write block 102 len 8192 data: 19 20 26/30 Read block 243 len 20480 27/30 Read block 110 len 4096 28/30 Read block 120 len 4096 29/30 Read block 520 len 24576 30/30 Write block 591 len 28672 data: 9 8 8 7 9 4 4 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:41.109732939Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:41.109990431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1848,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 306 306 306 F F F 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 178 168 178 311 291 311 F F F <--- 4 080-099 178 171 178 311 296 311 F F F <--- 5 100-119 176 176 176 308 308 308 T F T <--- 6 120-139 174 174 174 301 301 301 F F F 7 140-159 178 174 178 310 300 310 F T F <--- 8 160-179 178 176 178 311 306 311 F F F <--- 9 180-199 178 169 178 311 293 311 F F F <--- 10 200-219 178 176 178 311 307 311 F F F <--- 11 220-239 169 169 169 293 293 293 F F F 12 240-259 176 176 176 307 307 307 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 178 177 178 311 309 311 F F F <--- 15 300-319 178 172 178 311 297 311 F F F <--- 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 176 176 176 304 304 304 F F F 26 520-539 178 176 178 311 304 311 F F F <--- 27 540-559 177 177 177 309 309 309 F F F 28 560-579 178 177 178 311 309 311 F F F <--- 29 580-599 178 176 178 311 304 311 T F T <--- Max gen: 178, Max flush: 311 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 306 306 306 F F F 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 179 179 179 312 312 312 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 178 178 178 310 310 310 F F F 8 160-179 178 178 178 311 311 311 F F F 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 169 169 169 293 293 293 F F F 12 240-259 176 176 176 307 307 307 F F F 13 260-279 173 173 173 298 298 298 F F F 14 280-299 178 178 178 311 311 311 F F F 15 300-319 178 178 178 311 311 311 F F F 16 320-339 173 173 173 298 298 298 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 176 176 176 304 304 304 F F F 26 520-539 178 178 178 311 311 311 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 178 178 178 311 311 311 F F F 29 580-599 179 179 179 312 312 312 F F F Max gen: 179, Max flush: 312 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.353867667Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.362350944Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.362695058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.362731539Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.362741349Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.362753359Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Crucible 86ebff0f-ef1a-49a4-aa1d-1085a2df8f28 has session id: 881d0a17-94e0-4747-bc6d-624dc895994b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363061072Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Upstairs opts: Upstairs UUID: 86ebff0f-ef1a-49a4-aa1d-1085a2df8f28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363103993Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Crucible stats registered with UUID: 86ebff0f-ef1a-49a4-aa1d-1085a2df8f28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363114753Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:45.363124113Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363318015Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363350266Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363361486Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363468577Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363487817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"86ebff0f-ef1a-49a4-aa1d-1085a2df8f28 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363514428Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363524628Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363534858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363544928Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363554888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363565208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363575658Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65217)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363625029Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63562)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363636549Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51004)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.363647279Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364266216Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364315197Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364384207Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.36457287Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.36459456Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[0]R flush_numbers[0..12]: [306, 301, 298, 311, 311, 312, 301, 310, 311, 311, 311, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364677581Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[0]R generation[0..12]: [176, 174, 173, 178, 178, 179, 174, 178, 178, 178, 178, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364689951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364701601Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[1]R flush_numbers[0..12]: [306, 301, 298, 311, 311, 312, 301, 310, 311, 311, 311, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364712801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[1]R generation[0..12]: [176, 174, 173, 178, 178, 179, 174, 178, 178, 178, 178, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364723761Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364735171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[2]R flush_numbers[0..12]: [306, 301, 298, 311, 311, 312, 301, 310, 311, 311, 311, 293]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364746222Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[2]R generation[0..12]: [176, 174, 173, 178, 178, 179, 174, 178, 178, 178, 178, 169]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364757112Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364768302Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364778212Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364788402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364798102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364807932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"":"downstairs","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364818252Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364829873Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"0","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364840073Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"1","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364850023Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"client":"2","":"downstairs","session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364860443Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364870173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"86ebff0f-ef1a-49a4-aa1d-1085a2df8f28 is now active with session: 881d0a17-94e0-4747-bc6d-624dc895994b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364880623Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364889823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"session_id":"881d0a17-94e0-4747-bc6d-624dc895994b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364899323Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Activated sub_volume 86ebff0f-ef1a-49a4-aa1d-1085a2df8f28","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364907743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.364916254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.365360779Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} 01/30 Write block 242 len 24576 data: 15 12 11 12 14 15 02/30 Read block 227 len 4096 03/30 Read block 354 len 24576 04/30 Read block 197 len 24576 05/30 Write block 101 len 12288 data: 18 20 21 06/30 Write block 157 len 36864 data: 11 11 12 14 11 11 12 13 14 07/30 Read block 506 len 12288 08/30 Read block 360 len 4096 09/30 Write block 245 len 4096 data: 13 10/30 Flush 11/30 Write block 230 len 16384 data: 18 18 18 18 12/30 Read block 423 len 20480 13/30 Read block 444 len 28672 14/30 Write block 242 len 36864 data: 16 13 12 14 15 16 15 14 17 15/30 Flush 16/30 Write block 10 len 32768 data: 13 13 12 11 15 15 13 14 17/30 Read block 102 len 36864 18/30 Write block 570 len 20480 data: 8 10 11 14 18 19/30 Read block 459 len 40960 20/30 Read block 490 len 16384 21/30 Write block 242 len 36864 data: 17 14 13 15 16 17 16 15 18 22/30 Read block 75 len 40960 23/30 Read block 573 len 36864 24/30 Write block 326 len 12288 data: 11 12 13 25/30 Write block 259 len 36864 data: 20 18 19 21 22 19 20 18 16 26/30 Read block 326 len 8192 27/30 Write block 516 len 36864 data: 12 12 12 10 14 12 12 11 12 28/30 Read block 87 len 16384 29/30 Read block 349 len 36864 30/30 Read block 328 len 24576 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 1 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:45.374877788Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:45.375154271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1856,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 306 306 306 T T F <--- 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 180 180 179 313 313 312 F F F <--- 6 120-139 174 174 174 301 301 301 F F F 7 140-159 180 180 178 313 313 310 F F F <--- 8 160-179 180 180 178 313 313 311 F F F <--- 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 180 180 169 314 314 293 F F F <--- 12 240-259 180 180 176 314 314 307 T T T <--- 13 260-279 173 173 173 298 298 298 T T F <--- 14 280-299 178 178 178 311 311 311 F F F 15 300-319 178 178 178 311 311 311 F F F 16 320-339 173 173 173 298 298 298 T T F <--- 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 176 176 176 304 304 304 T T F <--- 26 520-539 178 178 178 311 311 311 T T F <--- 27 540-559 177 177 177 309 309 309 F F F 28 560-579 178 178 178 311 311 311 T T F <--- 29 580-599 179 179 179 312 312 312 F F F Max gen: 180, Max flush: 314 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 315 315 315 F F F 1 020-039 174 174 174 301 301 301 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 178 178 178 311 311 311 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 180 180 180 313 313 313 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 180 180 180 313 313 313 F F F 8 160-179 180 180 180 313 313 313 F F F 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 180 180 180 314 314 314 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 178 178 178 311 311 311 F F F 15 300-319 178 178 178 311 311 311 F F F 16 320-339 181 181 181 315 315 315 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F F F 25 500-519 181 181 181 315 315 315 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 181 181 181 315 315 315 F F F 29 580-599 179 179 179 312 312 312 F F F Max gen: 181, Max flush: 315 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.617696451Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:49.625120996Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.625200697Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.63247485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.632733753Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.632757484Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.632768604Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.632780324Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Crucible d644d2c0-c805-4d3a-a301-682b869470de has session id: dc4e99b1-f380-4892-81dd-842bd1ec1356","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633143668Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Upstairs opts: Upstairs UUID: d644d2c0-c805-4d3a-a301-682b869470de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633173738Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Crucible stats registered with UUID: d644d2c0-c805-4d3a-a301-682b869470de","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633185158Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:49.633194819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633394141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633424351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633435681Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633557603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633584803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"d644d2c0-c805-4d3a-a301-682b869470de active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633597653Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633606993Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633617233Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633627403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633639254Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633649824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633660174Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"ds_connection connected from Ok(127.0.0.1:41419)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633708524Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"ds_connection connected from Ok(127.0.0.1:61035)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633720065Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"ds_connection connected from Ok(127.0.0.1:49712)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.633731075Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634412082Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634465513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634487453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634705276Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634740606Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[0]R flush_numbers[0..12]: [315, 301, 298, 311, 311, 313, 301, 313, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634803107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[0]R generation[0..12]: [181, 174, 173, 178, 178, 180, 174, 180, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634814627Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634825907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[1]R flush_numbers[0..12]: [315, 301, 298, 311, 311, 313, 301, 313, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634839237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[1]R generation[0..12]: [181, 174, 173, 178, 178, 180, 174, 180, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634850237Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634861478Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[2]R flush_numbers[0..12]: [315, 301, 298, 311, 311, 313, 301, 313, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634872268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[2]R generation[0..12]: [181, 174, 173, 178, 178, 180, 174, 180, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634883058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634894258Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634903988Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634913998Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634923828Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634933728Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"":"downstairs","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634943739Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634952989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"0","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634962989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"1","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634972899Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"client":"2","":"downstairs","session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634983149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.634993839Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"d644d2c0-c805-4d3a-a301-682b869470de is now active with session: dc4e99b1-f380-4892-81dd-842bd1ec1356","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.635004299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.635013519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"session_id":"dc4e99b1-f380-4892-81dd-842bd1ec1356"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.635023009Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Activated sub_volume d644d2c0-c805-4d3a-a301-682b869470de","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.63503129Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.63503989Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.635441284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} 01/30 Write block 264 len 36864 data: 20 21 19 17 16 13 11 12 12 02/30 Read block 12 len 12288 03/30 Write block 499 len 36864 data: 15 14 14 14 13 12 12 12 11 04/30 Write block 515 len 8192 data: 12 13 05/30 Read block 245 len 28672 06/30 Read block 258 len 20480 07/30 Write block 156 len 16384 data: 11 12 12 13 08/30 Read block 222 len 40960 09/30 Read block 364 len 16384 10/30 Read block 335 len 40960 11/30 Write block 401 len 20480 data: 11 12 12 15 14 12/30 Write block 17 len 40960 data: 15 14 14 15 16 14 10 9 10 9 13/30 Read block 14 len 32768 14/30 Read block 21 len 12288 15/30 Read block 499 len 24576 16/30 Read block 184 len 32768 17/30 Read block 114 len 12288 18/30 Read block 339 len 4096 19/30 Read block 469 len 32768 20/30 Write block 22 len 8192 data: 15 11 21/30 Read block 373 len 40960 22/30 Write block 64 len 32768 data: 8 13 11 11 9 7 11 14 23/30 Read block 562 len 28672 24/30 Read block 564 len 8192 25/30 Read block 65 len 16384 26/30 Write block 145 len 16384 data: 9 9 10 9 27/30 Read block 377 len 8192 28/30 Read block 385 len 24576 29/30 Write block 309 len 4096 data: 10 30/30 Read block 310 len 4096 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:49.64382819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:49.644113014Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1864,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 315 315 315 F T T <--- 1 020-039 174 174 174 301 301 301 F T T <--- 2 040-059 173 173 173 298 298 298 F F F 3 060-079 178 178 178 311 311 311 F T T <--- 4 080-099 178 178 178 311 311 311 F F F 5 100-119 180 180 180 313 313 313 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 180 180 180 313 313 313 F T T <--- 8 160-179 180 180 180 313 313 313 F F F 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 180 180 180 314 314 314 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 T T T 14 280-299 178 178 178 311 311 311 F F F 15 300-319 178 178 178 311 311 311 F T T <--- 16 320-339 181 181 181 315 315 315 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 173 173 173 298 298 298 F T T <--- 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 174 174 174 301 301 301 F T T <--- 25 500-519 181 181 181 315 315 315 F T T <--- 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 181 181 181 315 315 315 F F F 29 580-599 179 179 179 312 312 312 F F F Max gen: 181, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 182 182 182 316 316 316 F F F 2 040-059 173 173 173 298 298 298 F F F 3 060-079 182 182 182 316 316 316 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 180 180 180 313 313 313 F F F 6 120-139 174 174 174 301 301 301 F F F 7 140-159 182 182 182 316 316 316 F F F 8 160-179 180 180 180 313 313 313 F F F 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 180 180 180 314 314 314 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 182 182 182 316 316 316 F F F 14 280-299 178 178 178 311 311 311 F F F 15 300-319 182 182 182 316 316 316 F F F 16 320-339 181 181 181 315 315 315 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 182 182 182 316 316 316 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 182 182 182 316 316 316 F F F 25 500-519 182 182 182 316 316 316 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 181 181 181 315 315 315 F F F 29 580-599 179 179 179 312 312 312 F F F Max gen: 182, Max flush: 316 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.891472938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:53.898931983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.899006224Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906073245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906337158Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906360629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906369389Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906381299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Crucible 8b16c9fe-643e-4af0-a17b-a60dad4f5100 has session id: 1fb45919-e51b-440b-aeb8-c4291e02351b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906750123Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Upstairs opts: Upstairs UUID: 8b16c9fe-643e-4af0-a17b-a60dad4f5100, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906783373Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Crucible stats registered with UUID: 8b16c9fe-643e-4af0-a17b-a60dad4f5100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906794823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:53.906804574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.906982246Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907021726Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907033186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907158408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907182858Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"8b16c9fe-643e-4af0-a17b-a60dad4f5100 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907212758Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907222518Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907232748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907242709Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907252609Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907263109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907273449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36019)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907446821Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51820)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907473771Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39842)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.907485431Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908017107Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908036058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908090848Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908349711Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908384742Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[0]R flush_numbers[0..12]: [316, 316, 298, 316, 311, 313, 301, 316, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908455962Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[0]R generation[0..12]: [182, 182, 173, 182, 178, 180, 174, 182, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908467803Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908479213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[1]R flush_numbers[0..12]: [316, 316, 298, 316, 311, 313, 301, 316, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908490183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[1]R generation[0..12]: [182, 182, 173, 182, 178, 180, 174, 182, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908501213Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908512473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[2]R flush_numbers[0..12]: [316, 316, 298, 316, 311, 313, 301, 316, 313, 311, 311, 314]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908525713Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[2]R generation[0..12]: [182, 182, 173, 182, 178, 180, 174, 182, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908536623Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908547814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908557704Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908567794Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908577444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908587194Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"":"downstairs","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908597324Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908606574Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"0","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908616464Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"1","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908626514Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"client":"2","":"downstairs","session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908636645Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908646285Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"8b16c9fe-643e-4af0-a17b-a60dad4f5100 is now active with session: 1fb45919-e51b-440b-aeb8-c4291e02351b","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908656485Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908665725Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"session_id":"1fb45919-e51b-440b-aeb8-c4291e02351b"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908675205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Activated sub_volume 8b16c9fe-643e-4af0-a17b-a60dad4f5100","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908684805Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.908693275Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.90909963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} 01/30 Write block 35 len 8192 data: 11 13 02/30 Read block 503 len 36864 03/30 Write block 299 len 36864 data: 12 15 17 18 19 18 16 13 10 04/30 Read block 140 len 24576 05/30 Read block 35 len 4096 06/30 Write block 567 len 40960 data: 15 16 12 9 11 12 15 19 20 23 07/30 Flush 08/30 Write block 487 len 20480 data: 11 10 11 15 16 09/30 Read block 28 len 40960 10/30 Read block 440 len 20480 11/30 Read block 402 len 28672 12/30 Write block 229 len 4096 data: 18 13/30 Read block 139 len 4096 14/30 Read block 365 len 40960 15/30 Write block 260 len 8192 data: 19 20 16/30 Write block 66 len 8192 data: 12 12 17/30 Read block 284 len 36864 18/30 Flush 19/30 Write block 161 len 8192 data: 12 12 20/30 Read block 7 len 8192 21/30 Read block 535 len 4096 22/30 Write block 326 len 36864 data: 12 13 14 12 13 12 16 16 14 23/30 Write block 61 len 20480 data: 10 10 9 9 14 24/30 Write block 9 len 36864 data: 12 14 14 13 12 16 16 14 16 25/30 Write block 30 len 20480 data: 11 10 10 11 11 26/30 Write block 487 len 12288 data: 12 11 12 27/30 Read block 456 len 20480 28/30 Write block 221 len 32768 data: 12 13 12 13 17 17 17 17 29/30 Write block 118 len 36864 data: 10 9 9 11 11 10 11 11 15 30/30 Write block 45 len 32768 data: 13 13 11 10 9 9 9 7 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 3 3 34 Done 2 27 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:53.917120322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:53.917421815Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1872,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F T T <--- 1 020-039 182 184 184 316 317 317 T T T <--- 2 040-059 173 173 173 298 298 298 F T T <--- 3 060-079 182 184 184 316 318 318 F T T <--- 4 080-099 178 178 178 311 311 311 F F F 5 100-119 180 180 180 313 313 313 F T T <--- 6 120-139 174 174 174 301 301 301 F T T <--- 7 140-159 182 182 182 316 316 316 F F F 8 160-179 180 180 180 313 313 313 F T T <--- 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 180 184 184 314 318 318 F T T <--- 12 240-259 181 181 181 315 315 315 F F F 13 260-279 182 184 184 316 318 318 F F F <--- 14 280-299 178 184 184 311 317 317 F F F <--- 15 300-319 182 184 184 316 317 317 F F F <--- 16 320-339 181 181 181 315 315 315 F T T <--- 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 182 182 182 316 316 316 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 182 184 184 316 318 318 F T T <--- 25 500-519 182 182 182 316 316 316 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 181 184 184 315 317 317 F F F <--- 29 580-599 179 179 179 312 312 312 F F F Max gen: 184, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 185 185 185 319 319 319 F F F 3 060-079 185 185 185 319 319 319 F F F 4 080-099 178 178 178 311 311 311 F F F 5 100-119 185 185 185 319 319 319 F F F 6 120-139 185 185 185 319 319 319 F F F 7 140-159 182 182 182 316 316 316 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 178 178 178 311 311 311 F F F 10 200-219 178 178 178 311 311 311 F F F 11 220-239 185 185 185 319 319 319 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 184 184 184 317 317 317 F F F 15 300-319 184 184 184 317 317 317 F F F 16 320-339 185 185 185 319 319 319 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 182 182 182 316 316 316 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 173 173 298 298 298 F F F 23 460-479 176 176 176 307 307 307 F F F 24 480-499 185 185 185 319 319 319 F F F 25 500-519 182 182 182 316 316 316 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 184 184 184 317 317 317 F F F 29 580-599 179 179 179 312 312 312 F F F Max gen: 185, Max flush: 319 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.175912638Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:58.183343513Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.183418004Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.190544226Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.190797978Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.190820769Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.190829719Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.190841659Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Crucible d15ffa27-f287-409e-b31c-1eeaa6cfef25 has session id: f1fec247-da96-4bc0-bc19-993ce1632de5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191245594Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Upstairs opts: Upstairs UUID: d15ffa27-f287-409e-b31c-1eeaa6cfef25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191275084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Crucible stats registered with UUID: d15ffa27-f287-409e-b31c-1eeaa6cfef25","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191286534Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:58.191298134Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191476426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191505437Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191516827Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191644938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191672198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"d15ffa27-f287-409e-b31c-1eeaa6cfef25 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191683869Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191693309Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191703469Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191713599Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191723639Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191734059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191744449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47175)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.1918447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37147)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191914411Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35622)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.191927521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192446867Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192466448Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192608419Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192747291Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192764561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[0]R flush_numbers[0..12]: [319, 319, 319, 319, 311, 319, 319, 316, 319, 311, 311, 319]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192898242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[0]R generation[0..12]: [185, 185, 185, 185, 178, 185, 185, 182, 185, 178, 178, 185]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192910183Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192921813Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[1]R flush_numbers[0..12]: [319, 319, 319, 319, 311, 319, 319, 316, 319, 311, 311, 319]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192932963Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[1]R generation[0..12]: [185, 185, 185, 185, 178, 185, 185, 182, 185, 178, 178, 185]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192944103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192955393Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[2]R flush_numbers[0..12]: [319, 319, 319, 319, 311, 319, 319, 316, 319, 311, 311, 319]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192966403Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[2]R generation[0..12]: [185, 185, 185, 185, 178, 185, 185, 182, 185, 178, 178, 185]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192977413Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.192988754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193000964Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193011314Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193021274Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193031084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"":"downstairs","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193041304Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193050474Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"0","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193060814Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"1","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193070874Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"client":"2","":"downstairs","session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193081025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193090725Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"d15ffa27-f287-409e-b31c-1eeaa6cfef25 is now active with session: f1fec247-da96-4bc0-bc19-993ce1632de5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193101195Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193110315Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"session_id":"f1fec247-da96-4bc0-bc19-993ce1632de5"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193119965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Activated sub_volume d15ffa27-f287-409e-b31c-1eeaa6cfef25","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193128515Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.193136975Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.19353409Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} 01/30 Write block 84 len 20480 data: 14 15 16 16 13 02/30 Read block 133 len 4096 03/30 Read block 188 len 32768 04/30 Write block 108 len 8192 data: 14 17 05/30 Read block 272 len 32768 06/30 Write block 144 len 12288 data: 7 10 10 07/30 Flush 08/30 Write block 481 len 24576 data: 8 9 11 10 9 9 09/30 Read block 312 len 20480 10/30 Read block 507 len 8192 11/30 Flush 12/30 Write block 237 len 36864 data: 13 13 13 10 13 18 15 14 16 13/30 Read block 380 len 36864 14/30 Read block 528 len 12288 15/30 Write block 457 len 24576 data: 10 13 16 16 15 13 16/30 Write block 590 len 12288 data: 11 10 9 17/30 Write block 61 len 36864 data: 11 11 10 10 15 13 13 10 8 18/30 Read block 259 len 16384 19/30 Read block 345 len 12288 20/30 Write block 71 len 32768 data: 15 14 16 13 13 15 15 15 21/30 Read block 398 len 4096 22/30 Flush 23/30 Flush 24/30 Write block 210 len 16384 data: 16 17 17 15 25/30 Flush 26/30 Write block 185 len 12288 data: 16 16 15 27/30 Read block 566 len 24576 28/30 Write block 470 len 8192 data: 12 11 29/30 Read block 526 len 24576 30/30 Write block 370 len 40960 data: 11 14 10 9 11 11 12 11 14 12 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 16 1 1 18 Done 3 18 18 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:26:58.202613454Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:26:58.202894797Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1880,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 185 185 185 319 319 319 F F F 3 060-079 185 186 186 319 322 322 F F F <--- 4 080-099 186 186 186 320 320 320 F F F 5 100-119 186 186 186 320 320 320 F F F 6 120-139 185 185 185 319 319 319 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 178 178 178 311 311 311 F T T <--- 10 200-219 178 186 186 311 324 324 F F F <--- 11 220-239 185 186 186 319 322 322 T F F <--- 12 240-259 181 186 186 315 322 322 T F F <--- 13 260-279 184 184 184 318 318 318 F F F 14 280-299 184 184 184 317 317 317 F F F 15 300-319 184 184 184 317 317 317 F F F 16 320-339 185 185 185 319 319 319 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 177 177 177 309 309 309 F T T <--- 19 380-399 177 177 177 309 309 309 F F F 20 400-419 182 182 182 316 316 316 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 173 186 186 298 322 322 F F F <--- 23 460-479 176 186 186 307 322 322 F T T <--- 24 480-499 186 186 186 321 321 321 F F F 25 500-519 182 182 182 316 316 316 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 184 184 184 317 317 317 F F F 29 580-599 179 186 186 312 322 322 F F F <--- Max gen: 186, Max flush: 324 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 185 185 185 319 319 319 F F F 3 060-079 186 186 186 322 322 322 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 186 186 186 320 320 320 F F F 6 120-139 185 185 185 319 319 319 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 186 186 186 324 324 324 F F F 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 184 184 184 317 317 317 F F F 15 300-319 184 184 184 317 317 317 F F F 16 320-339 185 185 185 319 319 319 F F F 17 340-359 173 173 173 298 298 298 F F F 18 360-379 187 187 187 325 325 325 F F F 19 380-399 177 177 177 309 309 309 F F F 20 400-419 182 182 182 316 316 316 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 187 187 187 325 325 325 F F F 24 480-499 186 186 186 321 321 321 F F F 25 500-519 182 182 182 316 316 316 F F F 26 520-539 181 181 181 315 315 315 F F F 27 540-559 177 177 177 309 309 309 F F F 28 560-579 184 184 184 317 317 317 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 187, Max flush: 325 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.444551247Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453287847Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453587481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453612141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453622961Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453634921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Crucible e81ab0dd-3424-4bfc-8723-8b9e26aec3e8 has session id: 0832bbad-9e16-4dbe-8da4-ae44c394520f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.453988015Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Upstairs opts: Upstairs UUID: e81ab0dd-3424-4bfc-8723-8b9e26aec3e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454064626Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Crucible stats registered with UUID: e81ab0dd-3424-4bfc-8723-8b9e26aec3e8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454077036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:02.454086786Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454261808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454277548Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454287749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.4544465Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454459981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"e81ab0dd-3424-4bfc-8723-8b9e26aec3e8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454469971Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454479061Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454489211Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454501491Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454511481Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454521981Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454532351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38748)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454589512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61439)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454600832Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61270)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.454611582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455403671Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455425992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455442142Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455762685Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455772796Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[0]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 320, 319, 320, 319, 325, 324, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455814716Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[0]R generation[0..12]: [185, 185, 185, 186, 186, 186, 185, 186, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455826626Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455838296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[1]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 320, 319, 320, 319, 325, 324, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455851707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[1]R generation[0..12]: [185, 185, 185, 186, 186, 186, 185, 186, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455862807Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455874367Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[2]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 320, 319, 320, 319, 325, 324, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455885427Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[2]R generation[0..12]: [185, 185, 185, 186, 186, 186, 185, 186, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455896327Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455907697Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455917407Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455927547Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455937397Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455947228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"":"downstairs","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455957578Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455966808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"0","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455977078Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"1","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455987228Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"client":"2","":"downstairs","session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.455998598Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456008218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"e81ab0dd-3424-4bfc-8723-8b9e26aec3e8 is now active with session: 0832bbad-9e16-4dbe-8da4-ae44c394520f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456018598Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456027729Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"session_id":"0832bbad-9e16-4dbe-8da4-ae44c394520f"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456037169Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Activated sub_volume e81ab0dd-3424-4bfc-8723-8b9e26aec3e8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456045489Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456053889Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.456503164Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} 01/30 Write block 555 len 40960 data: 14 17 20 21 19 13 13 14 14 13 02/30 Read block 560 len 24576 03/30 Write block 520 len 32768 data: 15 13 13 12 13 13 13 11 04/30 Read block 388 len 32768 05/30 Read block 456 len 4096 06/30 Read block 542 len 36864 07/30 Write block 386 len 36864 data: 18 20 18 15 14 15 15 13 11 08/30 Read block 60 len 28672 09/30 Read block 561 len 12288 10/30 Write block 392 len 32768 data: 16 14 12 11 13 13 15 9 11/30 Write block 545 len 8192 data: 21 24 12/30 Write block 499 len 36864 data: 16 15 15 15 14 13 13 13 12 13/30 Read block 359 len 12288 14/30 Write block 479 len 16384 data: 11 11 9 10 15/30 Read block 12 len 24576 16/30 Read block 64 len 32768 17/30 Write block 129 len 8192 data: 18 15 18/30 Write block 409 len 24576 data: 9 9 12 13 13 15 19/30 Write block 368 len 36864 data: 10 12 12 15 11 10 12 12 13 20/30 Write block 347 len 28672 data: 17 15 15 10 12 10 10 21/30 Write block 373 len 20480 data: 11 13 13 14 12 22/30 Read block 472 len 40960 23/30 Write block 405 len 8192 data: 15 13 24/30 Write block 118 len 40960 data: 11 10 10 12 12 11 12 12 16 16 25/30 Read block 333 len 28672 26/30 Flush 27/30 Write block 206 len 16384 data: 11 11 13 14 28/30 Write block 500 len 16384 data: 16 16 16 15 29/30 Read block 508 len 16384 30/30 Write block 472 len 32768 data: 12 13 17 15 14 11 11 12 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:02.465631368Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:02.465916092Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1888,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 185 185 185 319 319 319 F F F 3 060-079 186 186 186 322 322 322 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 188 186 188 326 320 326 F F F <--- 6 120-139 188 185 188 326 319 326 F F F <--- 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 186 186 186 324 324 324 T F T <--- 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 184 184 184 317 317 317 F F F 15 300-319 184 184 184 317 317 317 F F F 16 320-339 185 185 185 319 319 319 F F F 17 340-359 188 173 188 326 298 326 F F F <--- 18 360-379 188 187 188 326 325 326 F F F <--- 19 380-399 188 177 188 326 309 326 F F F <--- 20 400-419 188 182 188 326 316 326 F F F <--- 21 420-439 173 173 173 298 298 298 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 188 187 188 326 325 326 T F T <--- 24 480-499 188 186 188 326 321 326 F F F <--- 25 500-519 188 182 188 326 316 326 T F T <--- 26 520-539 188 181 188 326 315 326 F F F <--- 27 540-559 188 177 188 326 309 326 F F F <--- 28 560-579 188 184 188 326 317 326 F F F <--- 29 580-599 186 186 186 322 322 322 F F F Max gen: 188, Max flush: 326 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 F F F 1 020-039 185 185 185 319 319 319 F F F 2 040-059 185 185 185 319 319 319 F F F 3 060-079 186 186 186 322 322 322 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 188 188 188 326 326 326 F F F 6 120-139 188 188 188 326 326 326 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 189 189 189 327 327 327 F F F 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 184 184 184 317 317 317 F F F 15 300-319 184 184 184 317 317 317 F F F 16 320-339 185 185 185 319 319 319 F F F 17 340-359 188 188 188 326 326 326 F F F 18 360-379 188 188 188 326 326 326 F F F 19 380-399 188 188 188 326 326 326 F F F 20 400-419 188 188 188 326 326 326 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 186 186 186 322 322 322 F F F 23 460-479 189 189 189 327 327 327 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 189 189 189 327 327 327 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 326 326 326 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 189, Max flush: 327 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.713116385Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.721548582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.721849245Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.721877666Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.721887076Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.721899036Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Crucible b59eecd7-f4a9-440c-ba4a-8e8004e205b2 has session id: faab794f-d59e-42a7-948c-d9b2441d01a9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72228046Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Upstairs opts: Upstairs UUID: b59eecd7-f4a9-440c-ba4a-8e8004e205b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722320571Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Crucible stats registered with UUID: b59eecd7-f4a9-440c-ba4a-8e8004e205b2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722331791Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:06.722341701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722524473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722556173Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722567434Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722708205Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722725405Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"b59eecd7-f4a9-440c-ba4a-8e8004e205b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722739186Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722748426Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722758296Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722768386Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722778496Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722788886Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722799236Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63553)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722861337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37438)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722873117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:65026)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.722883857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.723567735Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.723609586Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.723656686Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.723926199Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.723948629Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[0]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 326, 326, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72395942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[0]R generation[0..12]: [185, 185, 185, 186, 186, 188, 188, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72397121Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72398268Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[1]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 326, 326, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.7239936Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[1]R generation[0..12]: [185, 185, 185, 186, 186, 188, 188, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72401781Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72402946Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[2]R flush_numbers[0..12]: [319, 319, 319, 322, 320, 326, 326, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.72404284Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[2]R generation[0..12]: [185, 185, 185, 186, 186, 188, 188, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724053951Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724065441Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724075271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724085521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724095321Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724105041Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"":"downstairs","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724115171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724124301Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"0","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724134512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"1","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724144632Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"client":"2","":"downstairs","session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724155002Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724164512Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"b59eecd7-f4a9-440c-ba4a-8e8004e205b2 is now active with session: faab794f-d59e-42a7-948c-d9b2441d01a9","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724174932Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724184102Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"session_id":"faab794f-d59e-42a7-948c-d9b2441d01a9"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724193712Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Activated sub_volume b59eecd7-f4a9-440c-ba4a-8e8004e205b2","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724203462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724211992Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.724614037Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} 01/30 Write block 272 len 20480 data: 13 15 13 16 17 02/30 Read block 324 len 28672 03/30 Read block 164 len 16384 04/30 Write block 497 len 8192 data: 12 15 05/30 Write block 4 len 4096 data: 8 06/30 Write block 311 len 36864 data: 13 11 11 10 11 13 13 11 10 07/30 Read block 538 len 20480 08/30 Read block 94 len 12288 09/30 Write block 413 len 16384 data: 14 16 16 19 10/30 Read block 259 len 24576 11/30 Write block 8 len 32768 data: 10 13 15 15 14 13 17 17 12/30 Write block 10 len 32768 data: 16 16 15 14 18 18 15 17 13/30 Read block 15 len 40960 14/30 Read block 310 len 32768 15/30 Read block 339 len 4096 16/30 Read block 180 len 28672 17/30 Write block 327 len 12288 data: 14 15 13 18/30 Read block 310 len 36864 19/30 Read block 518 len 16384 20/30 Write block 455 len 20480 data: 9 11 11 14 17 21/30 Write block 39 len 8192 data: 12 11 22/30 Write block 132 len 4096 data: 15 23/30 Read block 167 len 24576 24/30 Write block 291 len 4096 data: 12 25/30 Write block 325 len 36864 data: 12 13 15 16 14 14 13 17 17 26/30 Write block 55 len 16384 data: 13 11 9 10 27/30 Write block 105 len 32768 data: 21 20 15 15 18 12 13 12 28/30 Write block 495 len 16384 data: 9 11 13 16 29/30 Write block 382 len 12288 data: 13 14 14 30/30 Write block 322 len 16384 data: 13 13 12 13 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 30 38 Done 26 26 0 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:06.731778489Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:06.732054322Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1896,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 319 319 319 T T F <--- 1 020-039 185 185 185 319 319 319 T T F <--- 2 040-059 185 185 185 319 319 319 T T F <--- 3 060-079 186 186 186 322 322 322 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 188 188 188 326 326 326 T T F <--- 6 120-139 188 188 188 326 326 326 T T F <--- 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 189 189 189 327 327 327 F F F 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 184 184 184 318 318 318 T T F <--- 14 280-299 184 184 184 317 317 317 T T F <--- 15 300-319 184 184 184 317 317 317 T T F <--- 16 320-339 185 185 185 319 319 319 T T F <--- 17 340-359 188 188 188 326 326 326 F F F 18 360-379 188 188 188 326 326 326 F F F 19 380-399 188 188 188 326 326 326 T T F <--- 20 400-419 188 188 188 326 326 326 T T F <--- 21 420-439 173 173 173 298 298 298 F F F 22 440-459 186 186 186 322 322 322 T T F <--- 23 460-479 189 189 189 327 327 327 F F F 24 480-499 188 188 188 326 326 326 T T F <--- 25 500-519 189 189 189 327 327 327 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 326 326 326 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 189, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 186 186 186 322 322 322 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 190 190 190 328 328 328 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 185 185 319 319 319 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 189 189 189 327 327 327 F F F 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 190 190 190 328 328 328 F F F 14 280-299 190 190 190 328 328 328 F F F 15 300-319 190 190 190 328 328 328 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 188 188 188 326 326 326 F F F 18 360-379 188 188 188 326 326 326 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 173 173 173 298 298 298 F F F 22 440-459 190 190 190 328 328 328 F F F 23 460-479 189 189 189 327 327 327 F F F 24 480-499 190 190 190 328 328 328 F F F 25 500-519 189 189 189 327 327 327 F F F 26 520-539 188 188 188 326 326 326 F F F 27 540-559 188 188 188 326 326 326 F F F 28 560-579 188 188 188 326 326 326 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 190, Max flush: 328 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.982252569Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:10.989771965Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.989847006Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99718566Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997435163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997459153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997468443Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997480444Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Crucible 9e8afcfb-6d5e-49d3-8476-6b2173f25722 has session id: f9ba93ee-6e8f-44b2-a101-da0ed079e7af","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997828888Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Upstairs opts: Upstairs UUID: 9e8afcfb-6d5e-49d3-8476-6b2173f25722, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997871378Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Crucible stats registered with UUID: 9e8afcfb-6d5e-49d3-8476-6b2173f25722","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.997882348Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:10.997894538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99807906Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998099171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998113111Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998235392Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998254272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"9e8afcfb-6d5e-49d3-8476-6b2173f25722 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998272353Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998281673Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998291633Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998301583Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998311453Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998321823Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998332103Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"ds_connection connected from Ok(127.0.0.1:59600)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998444395Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"ds_connection connected from Ok(127.0.0.1:51401)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998470805Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"ds_connection connected from Ok(127.0.0.1:62307)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.998527706Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999110592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999138793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999266834Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999409176Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999426676Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[0]R flush_numbers[0..12]: [328, 328, 328, 322, 320, 328, 328, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999574478Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 186, 186, 190, 190, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999586108Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999597538Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[1]R flush_numbers[0..12]: [328, 328, 328, 322, 320, 328, 328, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999608688Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 186, 186, 190, 190, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999619768Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999631148Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[2]R flush_numbers[0..12]: [328, 328, 328, 322, 320, 328, 328, 320, 319, 325, 327, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999642188Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 186, 186, 190, 190, 186, 185, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999653058Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999664509Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999676299Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999686519Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999696249Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999706009Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"":"downstairs","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999716329Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999725449Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"0","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999735699Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"1","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.999745819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"client":"2","":"downstairs","session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99975622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99976603Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"9e8afcfb-6d5e-49d3-8476-6b2173f25722 is now active with session: f9ba93ee-6e8f-44b2-a101-da0ed079e7af","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99977624Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99978543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"session_id":"f9ba93ee-6e8f-44b2-a101-da0ed079e7af"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99979506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Activated sub_volume 9e8afcfb-6d5e-49d3-8476-6b2173f25722","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99980361Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:10.99981208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:11.000268595Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} 01/30 Write block 277 len 32768 data: 20 24 20 17 13 13 12 14 02/30 Flush 03/30 Write block 370 len 20480 data: 13 16 12 12 14 04/30 Read block 233 len 4096 05/30 Write block 490 len 36864 data: 16 17 13 12 11 10 12 14 17 06/30 Read block 225 len 40960 07/30 Write block 425 len 36864 data: 7 9 9 10 11 15 13 13 14 08/30 Write block 207 len 16384 data: 12 14 15 17 09/30 Read block 229 len 32768 10/30 Write block 261 len 32768 data: 21 22 23 21 22 20 18 17 11/30 Read block 497 len 40960 12/30 Write block 313 len 24576 data: 12 11 12 14 14 12 13/30 Read block 202 len 24576 14/30 Write block 165 len 28672 data: 15 14 15 12 13 11 13 15/30 Read block 6 len 4096 16/30 Read block 35 len 16384 17/30 Read block 434 len 16384 18/30 Write block 535 len 36864 data: 18 20 19 17 14 16 15 13 16 19/30 Write block 434 len 24576 data: 14 17 17 17 16 17 20/30 Write block 504 len 16384 data: 14 14 14 13 21/30 Write block 573 len 8192 data: 16 20 22/30 Read block 346 len 4096 23/30 Flush 24/30 Write block 495 len 36864 data: 11 13 15 18 17 17 17 17 16 25/30 Read block 147 len 12288 26/30 Read block 166 len 16384 27/30 Read block 421 len 24576 28/30 Read block 274 len 24576 29/30 Write block 65 len 20480 data: 16 14 14 11 9 30/30 Write block 451 len 20480 data: 8 9 9 6 10 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 5 Done Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 1 26 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:11.009986137Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:11.01027533Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1904,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 186 186 186 322 322 322 F T T <--- 4 080-099 186 186 186 320 320 320 F F F 5 100-119 190 190 190 328 328 328 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 185 192 192 319 330 330 F F F <--- 9 180-199 187 187 187 325 325 325 F F F 10 200-219 189 192 192 327 330 330 F F F <--- 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 192 192 192 329 330 330 F F F <--- 14 280-299 192 192 192 329 329 329 F F F 15 300-319 190 192 192 328 330 330 F F F <--- 16 320-339 190 190 190 328 328 328 F F F 17 340-359 188 188 188 326 326 326 F F F 18 360-379 188 192 192 326 330 330 T F F <--- 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 173 192 192 298 330 330 F F F <--- 22 440-459 190 190 190 328 328 328 F T T <--- 23 460-479 189 189 189 327 327 327 F F F 24 480-499 190 192 192 328 330 330 F T T <--- 25 500-519 189 192 192 327 330 330 F T T <--- 26 520-539 188 192 192 326 330 330 F F F <--- 27 540-559 188 192 192 326 330 330 F F F <--- 28 560-579 188 192 192 326 330 330 F F F <--- 29 580-599 186 186 186 322 322 322 F F F Max gen: 192, Max flush: 330 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 193 193 193 331 331 331 F F F 4 080-099 186 186 186 320 320 320 F F F 5 100-119 190 190 190 328 328 328 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 192 192 192 330 330 330 F F F 11 220-239 186 186 186 322 322 322 F F F 12 240-259 186 186 186 322 322 322 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 192 192 192 329 329 329 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 188 188 188 326 326 326 F F F 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 192 192 192 330 330 330 F F F 22 440-459 193 193 193 331 331 331 F F F 23 460-479 189 189 189 327 327 327 F F F 24 480-499 193 193 193 331 331 331 F F F 25 500-519 193 193 193 331 331 331 F F F 26 520-539 192 192 192 330 330 330 F F F 27 540-559 192 192 192 330 330 330 F F F 28 560-579 192 192 192 330 330 330 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 193, Max flush: 331 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.26847374Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27724334Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277548363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277586984Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277596634Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277608784Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Crucible 462bd025-9557-4829-8d60-22857de875b3 has session id: ab57171d-68f8-4a47-9e94-09a100ff52c8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277904018Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Upstairs opts: Upstairs UUID: 462bd025-9557-4829-8d60-22857de875b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277958408Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Crucible stats registered with UUID: 462bd025-9557-4829-8d60-22857de875b3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.277969528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:15.277979178Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278164151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278195171Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278206401Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278336402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278349743Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"462bd025-9557-4829-8d60-22857de875b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278365473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278374913Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278384863Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278395063Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278404983Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278415153Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278425754Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49576)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278507824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52591)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278534675Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45620)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.278546165Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279262033Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279280283Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279325864Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279644517Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279667218Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[0]R flush_numbers[0..12]: [328, 328, 328, 331, 320, 328, 328, 320, 330, 325, 330, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279680678Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 193, 186, 190, 190, 186, 192, 187, 192, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279692198Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279703748Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[1]R flush_numbers[0..12]: [328, 328, 328, 331, 320, 328, 328, 320, 330, 325, 330, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279714908Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 193, 186, 190, 190, 186, 192, 187, 192, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279725808Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279737079Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[2]R flush_numbers[0..12]: [328, 328, 328, 331, 320, 328, 328, 320, 330, 325, 330, 322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279747959Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 193, 186, 190, 190, 186, 192, 187, 192, 186]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279758879Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279770149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279780109Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279790189Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279800049Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279809849Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"":"downstairs","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279820059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27983048Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"0","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27984105Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"1","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27985117Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"client":"2","":"downstairs","session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27986144Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27987099Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"462bd025-9557-4829-8d60-22857de875b3 is now active with session: ab57171d-68f8-4a47-9e94-09a100ff52c8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27988141Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.2798907Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"session_id":"ab57171d-68f8-4a47-9e94-09a100ff52c8"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27990097Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Activated sub_volume 462bd025-9557-4829-8d60-22857de875b3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.27990919Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.279917861Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.280375586Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} 01/30 Write block 457 len 28672 data: 12 15 18 17 16 14 12 02/30 Write block 222 len 40960 data: 14 13 14 18 18 18 18 19 19 19 03/30 Flush 04/30 Write block 75 len 40960 data: 14 16 16 16 17 20 20 21 15 15 05/30 Read block 417 len 32768 06/30 Write block 450 len 4096 data: 9 07/30 Read block 34 len 12288 08/30 Write block 572 len 24576 data: 13 17 21 21 24 21 09/30 Read block 207 len 28672 10/30 Read block 567 len 24576 11/30 Read block 342 len 20480 12/30 Flush 13/30 Write block 97 len 32768 data: 18 16 14 16 19 21 22 18 14/30 Flush 15/30 Write block 249 len 36864 data: 16 19 16 15 15 12 14 17 14 16/30 Read block 21 len 24576 17/30 Read block 36 len 12288 18/30 Write block 479 len 8192 data: 13 12 19/30 Read block 560 len 36864 20/30 Write block 207 len 32768 data: 13 15 16 18 18 18 16 19 21/30 Read block 436 len 4096 22/30 Write block 484 len 20480 data: 11 10 10 13 12 23/30 Read block 410 len 24576 24/30 Write block 544 len 40960 data: 17 22 25 24 20 17 20 19 19 17 25/30 Write block 220 len 32768 data: 11 13 15 14 15 19 19 19 26/30 Write block 224 len 28672 data: 16 20 20 20 19 20 20 27/30 Read block 418 len 12288 28/30 Read block 590 len 20480 29/30 Write block 64 len 16384 data: 11 17 15 15 30/30 Write block 353 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) None Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:15.289944535Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:15.290259059Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1912,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 194 193 194 333 331 333 T F T <--- 4 080-099 194 186 194 334 320 334 F F F <--- 5 100-119 194 190 194 334 328 334 F F F <--- 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 192 192 192 330 330 330 T F T <--- 11 220-239 194 186 194 332 322 332 T F T <--- 12 240-259 186 186 186 322 322 322 T F T <--- 13 260-279 192 192 192 330 330 330 F F F 14 280-299 192 192 192 329 329 329 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 188 188 188 326 326 326 T F T <--- 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 192 192 192 330 330 330 F F F 22 440-459 194 193 194 333 331 333 F T F <--- 23 460-479 194 189 194 332 327 332 T T T <--- 24 480-499 193 193 193 331 331 331 T F T <--- 25 500-519 193 193 193 331 331 331 F F F 26 520-539 192 192 192 330 330 330 F F F 27 540-559 192 192 192 330 330 330 T F T <--- 28 560-579 194 192 194 333 330 333 F F F <--- 29 580-599 186 186 186 322 322 322 F F F Max gen: 194, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 190 190 190 328 328 328 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 195 195 195 335 335 335 F F F 4 080-099 194 194 194 334 334 334 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 195 195 195 335 335 335 F F F 11 220-239 195 195 195 335 335 335 F F F 12 240-259 195 195 195 335 335 335 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 192 192 192 329 329 329 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 192 192 192 330 330 330 F F F 22 440-459 194 194 194 333 333 333 F F F 23 460-479 195 195 195 335 335 335 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 193 193 193 331 331 331 F F F 26 520-539 192 192 192 330 330 330 F F F 27 540-559 195 195 195 335 335 335 F F F 28 560-579 194 194 194 333 333 333 F F F 29 580-599 186 186 186 322 322 322 F F F Max gen: 195, Max flush: 335 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.538295382Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54687149Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547166424Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547191084Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547199954Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547213924Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Crucible 395494ad-faef-49b0-8bcc-8d4061a476cc has session id: c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547639789Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Upstairs opts: Upstairs UUID: 395494ad-faef-49b0-8bcc-8d4061a476cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54766878Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Crucible stats registered with UUID: 395494ad-faef-49b0-8bcc-8d4061a476cc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54767986Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:19.54769246Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547891172Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547910782Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.547921202Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548073934Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548092644Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"395494ad-faef-49b0-8bcc-8d4061a476cc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548166235Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548175925Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548185995Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548196016Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548206056Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548217816Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548227826Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64783)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548267486Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57904)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548288747Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38505)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548300087Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548947824Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.548990005Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549133116Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549260528Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549279078Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[0]R flush_numbers[0..12]: [328, 328, 328, 335, 334, 334, 328, 320, 330, 325, 335, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54944757Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 195, 194, 194, 190, 186, 192, 187, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54945938Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54947062Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[1]R flush_numbers[0..12]: [328, 328, 328, 335, 334, 334, 328, 320, 330, 325, 335, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.54948132Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 195, 194, 194, 190, 186, 192, 187, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.5494943Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549506271Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[2]R flush_numbers[0..12]: [328, 328, 328, 335, 334, 334, 328, 320, 330, 325, 335, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549517151Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 195, 194, 194, 190, 186, 192, 187, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549527801Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549538851Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549548521Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549558571Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549568201Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549577841Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"":"downstairs","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549587941Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549596982Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"0","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549606732Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"1","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549616532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"client":"2","":"downstairs","session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549626612Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549635912Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"395494ad-faef-49b0-8bcc-8d4061a476cc is now active with session: c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549646092Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549656152Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"session_id":"c7fa30b3-f8e6-4ed6-a584-90c5c6c69b7f"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549665462Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Activated sub_volume 395494ad-faef-49b0-8bcc-8d4061a476cc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549673752Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.549682163Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.550100967Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} 01/30 Write block 212 len 20480 data: 19 17 20 17 15 02/30 Read block 206 len 32768 03/30 Write block 20 len 4096 data: 16 04/30 Read block 555 len 28672 05/30 Read block 596 len 12288 06/30 Read block 301 len 24576 07/30 Write block 573 len 36864 data: 18 22 22 25 22 22 22 22 22 08/30 Read block 63 len 32768 09/30 Read block 56 len 40960 10/30 Write block 467 len 28672 data: 6 6 9 13 12 13 14 11/30 Write block 250 len 16384 data: 20 17 16 16 12/30 Read block 452 len 40960 13/30 Read block 566 len 12288 14/30 Write block 537 len 28672 data: 20 18 15 17 16 14 17 15/30 Read block 520 len 8192 16/30 Read block 74 len 4096 17/30 Read block 344 len 32768 18/30 Read block 454 len 32768 19/30 Write block 294 len 12288 data: 12 11 14 20/30 Write block 61 len 36864 data: 12 12 11 12 18 16 16 12 10 21/30 Flush 22/30 Write block 24 len 16384 data: 10 11 10 9 23/30 Flush 24/30 Flush 25/30 Write block 468 len 16384 data: 7 10 14 13 26/30 Flush 27/30 Write block 565 len 4096 data: 15 28/30 Write block 143 len 28672 data: 8 8 11 11 11 10 9 29/30 Write block 541 len 4096 data: 17 30/30 Read block 451 len 32768 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:19.559009749Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:19.559295613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1920,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 196 196 190 337 337 328 F F F <--- 2 040-059 190 190 190 328 328 328 F F F 3 060-079 196 196 195 336 336 335 F F F <--- 4 080-099 194 194 194 334 334 334 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 186 186 186 320 320 320 T T F <--- 8 160-179 192 192 192 330 330 330 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 196 196 195 336 336 335 F F T <--- 11 220-239 195 195 195 335 335 335 F F F 12 240-259 196 196 195 336 336 335 F F F <--- 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 192 336 336 329 F F F <--- 15 300-319 192 192 192 330 330 330 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 192 192 192 330 330 330 F F F 22 440-459 194 194 194 333 333 333 F F F 23 460-479 196 196 195 339 339 335 F F F <--- 24 480-499 195 195 195 335 335 335 F F F 25 500-519 193 193 193 331 331 331 F F F 26 520-539 196 196 192 336 336 330 F F F <--- 27 540-559 196 196 195 336 336 335 T T F <--- 28 560-579 196 196 194 336 336 333 T T F <--- 29 580-599 196 196 186 336 336 322 F F F <--- Max gen: 196, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 196 196 196 337 337 337 F F F 2 040-059 190 190 190 328 328 328 F F F 3 060-079 196 196 196 336 336 336 F F F 4 080-099 194 194 194 334 334 334 F F F 5 100-119 194 194 194 334 334 334 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 197 197 197 340 340 340 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 187 187 187 325 325 325 F F F 10 200-219 196 196 196 336 336 336 F F F 11 220-239 195 195 195 335 335 335 F F F 12 240-259 196 196 196 336 336 336 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 336 336 336 F F F 15 300-319 192 192 192 330 330 330 F F F 16 320-339 190 190 190 328 328 328 F F F 17 340-359 195 195 195 335 335 335 F F F 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 192 192 192 330 330 330 F F F 22 440-459 194 194 194 333 333 333 F F F 23 460-479 196 196 196 339 339 339 F F F 24 480-499 195 195 195 335 335 335 F F F 25 500-519 193 193 193 331 331 331 F F F 26 520-539 196 196 196 336 336 336 F F F 27 540-559 197 197 197 340 340 340 F F F 28 560-579 197 197 197 340 340 340 F F F 29 580-599 196 196 196 336 336 336 F F F Max gen: 197, Max flush: 340 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.803035467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.811522774Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.811812077Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.811836147Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.811845208Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.811857138Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Crucible 0185fae3-5440-411d-be70-0dabb80ece5a has session id: 31973ec1-0ef1-4fda-bd7c-241564d394e0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812214132Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Upstairs opts: Upstairs UUID: 0185fae3-5440-411d-be70-0dabb80ece5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812258012Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Crucible stats registered with UUID: 0185fae3-5440-411d-be70-0dabb80ece5a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812269272Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:23.812281003Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812459085Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812474705Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812485025Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812601006Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812619506Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"0185fae3-5440-411d-be70-0dabb80ece5a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812654717Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812664447Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812674527Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812684467Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812694337Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812704707Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812714857Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49211)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812789918Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56211)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812815819Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58748)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.812827499Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813479966Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813506687Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813518817Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.81381592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.81383613Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[0]R flush_numbers[0..12]: [328, 337, 328, 336, 334, 334, 328, 340, 330, 325, 336, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813871921Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[0]R generation[0..12]: [190, 196, 190, 196, 194, 194, 190, 197, 192, 187, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813883701Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813895181Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[1]R flush_numbers[0..12]: [328, 337, 328, 336, 334, 334, 328, 340, 330, 325, 336, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813906351Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[1]R generation[0..12]: [190, 196, 190, 196, 194, 194, 190, 197, 192, 187, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813917331Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813928561Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[2]R flush_numbers[0..12]: [328, 337, 328, 336, 334, 334, 328, 340, 330, 325, 336, 335]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813939592Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[2]R generation[0..12]: [190, 196, 190, 196, 194, 194, 190, 197, 192, 187, 196, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813950412Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813961582Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813973622Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813983942Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.813993492Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814003242Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"":"downstairs","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814013402Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814022532Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"0","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814032473Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"1","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814042383Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"client":"2","":"downstairs","session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814052793Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814062543Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"0185fae3-5440-411d-be70-0dabb80ece5a is now active with session: 31973ec1-0ef1-4fda-bd7c-241564d394e0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814072913Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814081933Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"session_id":"31973ec1-0ef1-4fda-bd7c-241564d394e0"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814091553Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Activated sub_volume 0185fae3-5440-411d-be70-0dabb80ece5a","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814099843Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814108363Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.814493088Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} 01/30 Write block 478 len 40960 data: 12 14 13 10 11 12 12 11 11 14 02/30 Flush 03/30 Write block 317 len 8192 data: 15 13 04/30 Read block 119 len 36864 05/30 Flush 06/30 Write block 472 len 40960 data: 14 15 18 16 15 12 13 15 14 11 07/30 Flush 08/30 Write block 422 len 24576 data: 8 6 5 8 10 10 09/30 Read block 575 len 8192 10/30 Write block 543 len 20480 data: 18 18 23 26 25 11/30 Write block 198 len 8192 data: 17 18 12/30 Read block 510 len 40960 13/30 Read block 135 len 28672 14/30 Flush 15/30 Write block 54 len 32768 data: 9 14 12 10 11 13 10 13 16/30 Read block 153 len 12288 17/30 Read block 81 len 36864 18/30 Read block 462 len 8192 19/30 Flush 20/30 Write block 27 len 16384 data: 10 10 12 12 21/30 Read block 329 len 16384 22/30 Write block 481 len 28672 data: 12 12 13 13 12 12 15 23/30 Read block 252 len 8192 24/30 Read block 418 len 28672 25/30 Write block 315 len 32768 data: 13 15 16 14 11 10 13 14 26/30 Read block 534 len 32768 27/30 Write block 519 len 32768 data: 11 16 14 14 13 14 14 14 28/30 Write block 321 len 8192 data: 14 15 29/30 Write block 107 len 8192 data: 16 16 30/30 Write block 340 len 16384 data: 11 15 16 19 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 28 4 36 Done 24 0 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1001)) Some(JobId(1018)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:27:23.823634851Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928} {"msg":"Wrote out file \"/tmp/test_live_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:27:23.823933135Z","hostname":"w-01KX57DQGFNJ07NR5QMY2FD5YS","pid":1928,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 196 196 196 337 337 337 T F T <--- 2 040-059 198 190 198 345 328 345 F F F <--- 3 060-079 198 196 198 345 336 345 F F F <--- 4 080-099 194 194 194 334 334 334 F F F 5 100-119 194 194 194 334 334 334 T F T <--- 6 120-139 190 190 190 328 328 328 F F F 7 140-159 197 197 197 340 340 340 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 198 187 198 344 325 344 F F F <--- 10 200-219 196 196 196 336 336 336 F F F 11 220-239 195 195 195 335 335 335 F F F 12 240-259 196 196 196 336 336 336 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 336 336 336 F F F 15 300-319 198 192 198 342 330 342 T F T <--- 16 320-339 190 190 190 328 328 328 T F T <--- 17 340-359 195 195 195 335 335 335 T F T <--- 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 198 192 198 344 330 344 F F F <--- 22 440-459 194 194 194 333 333 333 F F F 23 460-479 198 198 198 343 341 343 F F F <--- 24 480-499 198 198 198 343 341 343 T F T <--- 25 500-519 193 193 193 331 331 331 T F T <--- 26 520-539 196 196 196 336 336 336 T F T <--- 27 540-559 198 197 198 344 340 344 F F F <--- 28 560-579 197 197 197 340 340 340 F F F 29 580-599 196 196 196 336 336 336 F F F Max gen: 198, Max flush: 345 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 328 328 328 F F F 1 020-039 199 199 199 346 346 346 F F F 2 040-059 198 198 198 345 345 345 F F F 3 060-079 198 198 198 345 345 345 F F F 4 080-099 194 194 194 334 334 334 F F F 5 100-119 199 199 199 346 346 346 F F F 6 120-139 190 190 190 328 328 328 F F F 7 140-159 197 197 197 340 340 340 F F F 8 160-179 192 192 192 330 330 330 F F F 9 180-199 198 198 198 344 344 344 F F F 10 200-219 196 196 196 336 336 336 F F F 11 220-239 195 195 195 335 335 335 F F F 12 240-259 196 196 196 336 336 336 F F F 13 260-279 192 192 192 330 330 330 F F F 14 280-299 196 196 196 336 336 336 F F F 15 300-319 199 199 199 346 346 346 F F F 16 320-339 199 199 199 346 346 346 F F F 17 340-359 199 199 199 346 346 346 F F F 18 360-379 192 192 192 330 330 330 F F F 19 380-399 190 190 190 328 328 328 F F F 20 400-419 190 190 190 328 328 328 F F F 21 420-439 198 198 198 344 344 344 F F F 22 440-459 194 194 194 333 333 333 F F F 23 460-479 198 198 198 343 343 343 F F F 24 480-499 199 199 199 346 346 346 F F F 25 500-519 199 199 199 346 346 346 F F F 26 520-539 199 199 199 346 346 346 F F F 27 540-559 198 198 198 344 344 344 F F F 28 560-579 197 197 197 340 340 340 F F F 29 580-599 196 196 196 336 336 336 F F F Max gen: 199, Max flush: 346 7:28 Test duration Test completed real 7:27.513852094 user 25.470593389 sys 14.430076510 trap 0.019317730 tflt 0.071454714 dflt 0.097712253 kflt 0.003656191 lock 11:57:15.390363718 slp 36:15.771791853 lat 2.293441439 stop 0.020008633